Dr. Fabian Joeres

@article{schwenderling_toward_2025,

title = {Toward structured abdominal examination training using augmented reality},

author = {L Schwenderling and L Hanke and U Holst and F Huettl and F Joeres and T Huber and C Hansen},

url = {https://doi.org/10.1007/s11548-024-03311-y},

doi = {10.1007/s11548-024-03311-y},

issn = {1861-6429},

year  = {2025},

date = {2025-05-01},

urldate = {2025-05-01},

journal = {International Journal of Computer Assisted Radiology and Surgery},

volume = {20},

number = {5},

pages = {949–958},

abstract = {Structured abdominal examination is an essential part of the medical curriculum and surgical training, requiring a blend of theory and practice from trainees. Current training methods, however, often do not provide adequate engagement, fail to address individual learning needs or do not cover rare diseases.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{gulamhussene_transfer-learning_2023,

title = {Transfer-learning is a key ingredient to fast deep learning-based 4D liver MRI reconstruction},

author = {G Gulamhussene and M Rak and O Bashkanov and F Joeres and J Omari and M Pech and C Hansen},

url = {https://www.nature.com/articles/s41598-023-38073-1},

doi = {10.1038/s41598-023-38073-1},

issn = {2045-2322},

year  = {2023},

date = {2023-07-01},

urldate = {2023-07-01},

journal = {Scientific Reports},

volume = {13},

number = {1},

pages = {11227},

abstract = {Time-resolved volumetric magnetic resonance imaging (4D MRI) could be used to address organ motion in image-guided interventions like tumor ablation. Current 4D reconstruction techniques are unsuitable for most interventional settings because they are limited to specific breathing phases, lack temporal/spatial resolution, and have long prior acquisitions or reconstruction times. Deep learning-based (DL) 4D MRI approaches promise to overcome these shortcomings but are sensitive to domain shift. This work shows that transfer learning (TL) combined with an ensembling strategy can help alleviate this key challenge. We evaluate four approaches: pre-trained models from the source domain, models directly trained from scratch on target domain data, models fine-tuned from a pre-trained model and an ensemble of fine-tuned models. For that the data base was split into 16 source and 4 target domain subjects. Comparing ensemble of fine-tuned models (N = 10) with directly learned models, we report significant improvements (P < 0.001) of the root mean squared error (RMSE) of up to 12% and the mean displacement (MDISP) of up to 17.5%. The smaller the target domain data amount, the larger the effect. This shows that TL + Ens significantly reduces beforehand acquisition time and improves reconstruction quality, rendering it a key component in making 4D MRI clinically feasible for the first time in the context of 4D organ motion models of the liver and beyond.},

note = {Publisher: Nature Publishing Group},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{schreiter_ar-supported_2022,

title = {AR-Supported Supervision of Conditional Autonomous Robots: Considerations for Pedicle Screw Placement in the Future},

author = {J Schreiter and D Schott and L Schwenderling and C Hansen and F Heinrich and F Joeres},

url = {https://www.mdpi.com/2313-433X/8/10/255},

doi = {10.3390/jimaging8100255},

issn = {2313-433X},

year  = {2022},

date = {2022-10-01},

urldate = {2022-10-01},

journal = {Journal of Imaging},

volume = {8},

number = {10},

pages = {255},

abstract = {Robotic assistance is applied in orthopedic interventions for pedicle screw placement (PSP). While current robots do not act autonomously, they are expected to have higher autonomy under surgeon supervision in the mid-term. Augmented reality (AR) is promising to support this supervision and to enable human–robot interaction (HRI). To outline a futuristic scenario for robotic PSP, the current workflow was analyzed through literature review and expert discussion. Based on this, a hypothetical workflow of the intervention was developed, which additionally contains the analysis of the necessary information exchange between human and robot. A video see-through AR prototype was designed and implemented. A robotic arm with an orthopedic drill mock-up simulated the robotic assistance. The AR prototype included a user interface to enable HRI. The interface provides data to facilitate understanding of the robot’s ”intentions”, e.g., patient-specific CT images, the current workflow phase, or the next planned robot motion. Two-dimensional and three-dimensional visualization illustrated patient-specific medical data and the drilling process. The findings of this work contribute a valuable approach in terms of addressing future clinical needs and highlighting the importance of AR support for HRI.},

note = {Publisher: Multidisciplinary Digital Publishing Institute},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{chheang_collaborative_2021,

title = {A collaborative virtual reality environment for liver surgery planning},

author = {V Chheang and P Saalfeld and F Joeres and C Boedecker and T Huber and F Huettl and H Lang and B Preim and C Hansen},

url = {https://www.sciencedirect.com/science/article/pii/S0097849321001400},

doi = {10.1016/j.cag.2021.07.009},

issn = {0097-8493},

year  = {2021},

date = {2021-10-01},

urldate = {2021-10-01},

journal = {Computers & Graphics},

volume = {99},

pages = {234–246},

abstract = {Surgical planning software is a key component in the treatment of tumor diseases. However, desktop-based systems provide only limited visualization and interaction opportunities. Moreover, collaborative planning among members of a surgical team is only possible to a limited extent. In this work, a collaborative virtual reality (VR) environment to assist liver surgeons in tumor surgery planning is presented. Our aim is to improve virtual resection planning between surgeons in a remote or co-located environment. The system allows surgeons to define and adjust virtual resections on patient-specific organ 3D surfaces and 2D image slices. Changes on both modalities are synchronized, which will enable surgeons to iterate and refine the resection surfaces quickly. In addition, a real-time risk map visualization is presented that displays safety margins around tumors. An evaluation performed by liver surgeons provides information on potential benefits, such as the possibility to visualize complex cases and assessing the safety-critical areas, applicability, and limitations for further improvement.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{wagner_ivroad_2021,

title = {iVRoad: Immersive virtual road crossing as an assessment tool for unilateral spatial neglect},

author = {S Wagner and J Belger and F Joeres and A Thöne-Otto and C Hansen and B Preim and P Saalfeld},

url = {https://www.sciencedirect.com/science/article/pii/S009784932100128X},

doi = {10.1016/j.cag.2021.06.013},

issn = {0097-8493},

year  = {2021},

date = {2021-10-01},

urldate = {2021-10-01},

journal = {Computers & Graphics},

volume = {99},

pages = {70–82},

abstract = {We developed a virtual road crossing assessment tool called iVRoad - immersive Virtual Road, which allows to put the patient into realistic road crossing situations and to record various parameters that can be used to quantify unilateral spatial neglect. We present a study with 18 stroke patients in which we evaluate our system with respect to usability, satisfaction, sense of presence and possible occurring cybersickness symptoms. Unilateral spatial neglect is a cognitive disturbance, often occurring after right hemispheric stroke. Conventional neuropsychological tests, such as paper-and-pencil tests, for assessing unilateral spatial neglect, often lack sensitivity. Especially in mild forms, symptoms can be seen in everyday life, but are hard to detect in formal testing. We examined patients with and without unilateral spatial neglect in order to identify parameters that could be feasible to separate these patient groups. Using everyday life tasks as a diagnostic instrument, however, is challenging because it is time-consuming, hard to control and to quantify. Computer-aided diagnostic systems are promising for analysing the behaviour of patients in detail. Modern virtual reality technology allows to place the patient in realistic situations. Especially situations in which patients often have difficulties or that are too dangerous in reality can be assessed with VR.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{joeres_laparoscopic_2021,

title = {Laparoscopic augmented reality registration for oncological resection site repair},

author = {F Joeres and T Mielke and C Hansen},

url = {https://doi.org/10.1007/s11548-021-02336-x},

doi = {10.1007/s11548-021-02336-x},

issn = {1861-6429},

year  = {2021},

date = {2021-09-01},

urldate = {2021-09-01},

journal = {International Journal of Computer Assisted Radiology and Surgery},

volume = {16},

number = {9},

pages = {1577–1586},

abstract = {Resection site repair during laparoscopic oncological surgery (e.g. laparoscopic partial nephrectomy) poses some unique challenges and opportunities for augmented reality (AR) navigation support. This work introduces an AR registration workflow that addresses the time pressure that is present during resection site repair.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{joeres_towards_2021,

title = {Towards natural 3D interaction for laparoscopic augmented reality registration},

author = {F Joeres and F Heinrich and D Schott and C Hansen},

url = {https://doi.org/10.1080/21681163.2020.1834877},

doi = {10.1080/21681163.2020.1834877},

issn = {2168-1163},

year  = {2021},

date = {2021-07-01},

urldate = {2021-07-01},

journal = {Computer Methods in Biomechanics and Biomedical Engineering: Imaging & Visualization},

volume = {9},

number = {4},

pages = {384–391},

abstract = {Augmented reality (AR) is a widely researched route for navigation support in laparoscopic surgery. Accurate registration is a crucial component for such AR systems. We introduce two methods for interactive registration that aim to be minimally invasive to the workflow and to mimic natural manipulation of 3D objects. The methods utilise spatially tracked laparoscopic tools to manipulate the virtual 3D content. We comparatively evaluated the methods against a reference, landmark-based registration method in a user study with 12 participants. We tested the methods for registration accuracy, time, and subjective usability perception. Our methods did not outperform the reference method on these parameters but showed promising results. The results indicate that our methods present no finalised solutions but that one of them is a promising approach for which we identified concrete improvement measures to be implemented in future research.},

note = {Publisher: Taylor & Francis 

_eprint: https://doi.org/10.1080/21681163.2020.1834877},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{joeres_audiovisual_2021,

title = {Audiovisual AR concepts for laparoscopic subsurface structure navigation},

author = {F Joeres and D Black and S Razavizadeh and C Hansen},

year  = {2021},

date = {2021-01-01},

urldate = {2021-01-01},

abstract = {The identification of subsurface structures during resection wound repair is a challenge during minimally invasive partial nephrectomy. Specifically, major blood vessels and branches of the urinary collecting system need to be localized under time pressure as target or risk structures during suture placement. This work presents concepts for AR visualization and auditory guidance based on tool position that support this task. We evaluated the concepts in a laboratory user study with a simplified, simulated task: The localization of subsurface target points in a healthy kidney phantom. We evaluated the task time, localization accuracy, and perceived workload for our concepts and a control condition without navigation support. The AR visualization improved the accuracy and perceived workload over the control condition. We observed similar, non-significant trends for the auditory display. Further, clinically realistic evaluation is pending. Our initial results indicate the potential benefits of our concepts in supporting laparoscopic resection wound repair.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{heinrich_comparison_2020,

title = {Comparison of Augmented Reality Display Techniques to Support Medical Needle Insertion},

author = {F Heinrich and L Schwenderling and F Joeres and K Lawonn and C Hansen},

url = {https://ieeexplore.ieee.org/abstract/document/9211732},

doi = {10.1109/TVCG.2020.3023637},

issn = {1941-0506},

year  = {2020},

date = {2020-12-01},

urldate = {2020-12-01},

journal = {IEEE Transactions on Visualization and Computer Graphics},

volume = {26},

number = {12},

pages = {3568–3575},

abstract = {Augmented reality (AR) may be a useful technique to overcome issues of conventionally used navigation systems supporting medical needle insertions, like increased mental workload and complicated hand-eye coordination. Previous research primarily focused on the development of AR navigation systems designed for specific displaying devices, but differences between employed methods have not been investigated before. To this end, a user study involving a needle insertion task was conducted comparing different AR display techniques with a monitor-based approach as baseline condition for the visualization of navigation information. A video see-through stationary display, an optical see-through head-mounted display and a spatial AR projector-camera-system were investigated in this comparison. Results suggest advantages of using projected navigation information in terms of lower task completion time, lower angular deviation and affirmative subjective participant feedback. Techniques requiring the intermediate view on screens, i.e. the stationary display and the baseline condition, showed less favorable results. Thus, benefits of providing AR navigation information compared to a conventionally used method could be identified. Significant objective measures results, as well as an identification of advantages and disadvantages of individual display techniques contribute to the development and design of improved needle navigation systems.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{gulamhussene_4d_2020,

title = {4D MRI: Robust sorting of free breathing MRI slices for use in interventional settings},

author = {G Gulamhussene and F Joeres and M Rak and M Pech and C Hansen},

url = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0235175},

doi = {10.1371/journal.pone.0235175},

issn = {1932-6203},

year  = {2020},

date = {2020-06-01},

urldate = {2020-06-01},

journal = {PLOS ONE},

volume = {15},

number = {6},

pages = {e0235175},

abstract = {Purpose We aim to develop a robust 4D MRI method for large FOVs enabling the extraction of irregular respiratory motion that is readily usable with all MRI machines and thus applicable to support a wide range of interventional settings. Method We propose a 4D MRI reconstruction method to capture an arbitrary number of breathing states. It uses template updates in navigator slices and search regions for fast and robust vessel cross-section tracking. It captures FOVs of 255 mm x 320 mm x 228 mm at a spatial resolution of 1.82 mm x 1.82 mm x 4mm and temporal resolution of 200ms. A total of 37 4D MRIs of 13 healthy subjects were reconstructed to validate the method. A quantitative evaluation of the reconstruction rate and speed of both the new and baseline method was performed. Additionally, a study with ten radiologists was conducted to assess the subjective reconstruction quality of both methods. Results Our results indicate improved mean reconstruction rates compared to the baseline method (79.4% vs. 45.5%) and improved mean reconstruction times (24s vs. 73s) per subject. Interventional radiologists perceive the reconstruction quality of our method as higher compared to the baseline (262.5 points vs. 217.5 points},

note = {Publisher: Public Library of Science},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{schott_lean-interaction_2020,

title = {Lean-Interaction: passive image manipulation in concurrent multitasking},

author = {D Schott and B Hatscher and F Joeres and M Gabele and S Hußlein and C Hansen},

year  = {2020},

date = {2020-01-01},

urldate = {2020-01-01},

abstract = {Complex bi-manual tasks often benefit from supporting visual information and guidance. Controlling the system that provides this information is a secondary task that forces the user to perform concurrent multitasking, which in turn may affect the main task performance. Interactions based on natural behavior are a promising solution to this challenge. We investigated the performance of these interactions in a hands-free image manipulation task during a primary manual task with an upright stance. Essential tasks were extracted from the example of clinical workflow and turned into an abstract simulation to gain general insights into how different interaction techniques impact the user’s performance and workload. The interaction techniques we compared were full-body movements, facial expression, gesture and speech input. We found that leaning as an interaction technique facilitates significantly faster image manipulation at lower subjective workloads than facial expression. Our results pave the way towards efficient, natural, hands-free interaction in a challenging multitasking environment.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{wagner_difficulty_2019,

title = {Difficulty factors for VR cognitive rehabilitation training – Crossing a virtual road},

author = {S Wagner and F Joeres and M Gabele and C Hansen and B Preim and P Saalfeld},

url = {https://www.sciencedirect.com/science/article/pii/S0097849319301037},

doi = {10.1016/j.cag.2019.06.009},

issn = {0097-8493},

year  = {2019},

date = {2019-10-01},

urldate = {2019-10-01},

journal = {Computers & Graphics},

volume = {83},

pages = {11–22},

abstract = {Patients with cognitive or visual impairments have problems in dealing with complex situations. During the rehabilitation process, it is important to confront the patient with (everyday) tasks that have increasing degrees of difficulty to improve their performance. Immersive virtual reality training offers the potential to create a better transfer to daily life than non-immersive computer training. In cooperation with two neuropsychologists, an immersive virtual environment (VE) was developed in which cognitive training in the form of safe road crossing decisions can be performed. We present the experimental exploration and evaluation of difficulty factors within such a VR-based cognitive rehabilitation program. Four difficulty factors were identified and compared (number of relevant traffic lanes, speed of vehicles, distance between vehicles, and number of vehicles). The combination of these difficulty factors resulted in 36 training scenarios. The impact of the factors on participant performance and subjective perception of scenario difficulty were evaluated with 60 healthy participants to estimate the impact of the four factors to a situation’s difficulty level. For the factors Relevant Lanes and Traffic Speed a clear influence on the perceived task difficulty could be determined. No clear influence could be found for the Gap Size. The Number of Vehicles had almost no effect on the perceived task difficulty. Finally, we asked two experienced neuropsychologists about the applicability of our developed system to patients, and they stated that the system is ready for a study on patients.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{joeres_how_2019,

title = {How well do software assistants for minimally invasive partial nephrectomy meet surgeon information needs? A cognitive task analysis and literature review study},

author = {F Joeres and D Schindele and M Luz and S Blaschke and N Russwinkel and M Schostak and C Hansen},

url = {https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0219920},

doi = {10.1371/journal.pone.0219920},

issn = {1932-6203},

year  = {2019},

date = {2019-07-01},

urldate = {2019-07-01},

journal = {PLOS ONE},

volume = {14},

number = {7},

pages = {e0219920},

abstract = {Introduction Intraoperative software assistance is gaining increasing importance in laparoscopic and robot-assisted surgery. Within the user-centred development process of such systems, the first question to be asked is: What information does the surgeon need and when does he or she need it? In this article, we present an approach to investigate these surgeon information needs for minimally invasive partial nephrectomy and compare these needs to the relevant surgical computer assistance literature. Materials and methods First, we conducted a literature-based hierarchical task analysis of the surgical procedure. This task analysis was taken as a basis for a qualitative in-depth interview study with nine experienced surgical urologists. The study employed a cognitive task analysis method to elicit surgeons' information needs during minimally invasive partial nephrectomy. Finally, a systematic literature search was conducted to review proposed software assistance solutions for minimally invasive partial nephrectomy. The review focused on what information the solutions present to the surgeon and what phase of the surgery they aim to support. Results The task analysis yielded a workflow description for minimally invasive partial nephrectomy. During the subsequent interview study, we identified three challenging phases of the procedure, which may particularly benefit from software assistance. These phases are I. Hilar and vascular management, II. Tumour excision, and III. Repair of the renal defects. Between these phases, 25 individual challenges were found which define the surgeon information needs. The literature review identified 34 relevant publications, all of which aim to support the surgeon in hilar and vascular management (phase I) or tumour excision (phase II). Conclusion The work presented in this article identified unmet surgeon information needs in minimally invasive partial nephrectomy. Namely, our results suggest that future solutions should address the repair of renal defects (phase III) or put more focus on the renal collecting system as a critical anatomical structure.},

note = {Publisher: Public Library of Science},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{heinrich_comparison_2019,

title = {Comparison of Projective Augmented Reality Concepts to Support Medical Needle Insertion},

author = {F Heinrich and F Joeres and K Lawonn and C Hansen},

url = {https://ieeexplore.ieee.org/document/8667734/},

doi = {10.1109/TVCG.2019.2903942},

issn = {1077-2626, 1941-0506, 2160-9306},

year  = {2019},

date = {2019-06-01},

urldate = {2019-06-01},

journal = {IEEE Transactions on Visualization and Computer Graphics},

volume = {25},

number = {6},

pages = {2157–2167},

abstract = {Augmented reality (AR) is a promising tool to improve instrument navigation in needle-based interventions. Limited research has been conducted regarding suitable navigation visualizations. In this work, three navigation concepts based on existing approaches were compared in a user study using a projective AR setup. Each concept was implemented with three different scales for accuracy-to-color mapping and two methods of navigation indicator scaling. Participants were asked to perform simulated needle insertion tasks with each of the resulting 18 prototypes. Insertion angle and insertion depth accuracies were measured and analyzed, as well as task completion time and participants’ subjectively perceived task difficulty. Results show a clear ranking of visualization concepts across variables. Less consistent results were obtained for the color and indicator scaling factors. Results suggest that logarithmic indicator scaling achieved better accuracy, but participants perceived it to be more difficult than linear scaling. With specific results for angle and depth accuracy, our study contributes to the future composition of improved navigation support and systems for precise needle insertion or similar applications.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}

@article{heinrich_eects_2019,

title = {Effects of Accuracy-to-Colour Mapping Scales on Needle Navigation Aids visualised by Projective Augmented Reality},

author = {F Heinrich and F Joeres and K Lawonn and C Hansen},

year  = {2019},

date = {2019-01-01},

urldate = {2019-01-01},

abstract = {Instrument navigation in needle-based interventions can benefit from augmented reality (AR) visualisation. Design aspects of these visualisations have been investigated to a limited degree. This work examined colourspecific parameters for AR instrument navigation, that have not been successfully researched before. Three different mapping methods to encode accuracy information to colour and two colour scales varying different colour channels were evaluated in a user study. Angular and depth accuracy of inserted needles were measured and task difficulty was subjectively rated. Result trends indicate benefits of mapping accuracy to discrete colours based on thresholds and using single hue colour scales that vary in the luminance or saturation channel. Yet, more research is required to validate the exposed indications. This work can constitute a valuable basis for this.},

keywords = {},

pubstate = {published},

tppubtype = {article}

}