@inproceedings{9652423956c64719a1d080668a69c178,
title = "Eye-Perspective View Management for Optical See-Through Head-Mounted Displays",
abstract = "Optical see-through (OST) head-mounted displays (HMDs) enable users to experience Augmented Reality (AR) support in the form of helpful real-world annotations. Unfortunately, the blend of the environment with virtual augmentations due to semitransparent OST displays often deteriorates the contrast and legibility of annotations. View management algorithms adapt the annotations' layout to improve legibility based on real-world information, typically captured by built-in HMD cameras. However, the camera views are different from the user's view through the OST display which decreases the final layout quality. We present eye-perspective view management that synthesizes high-fidelity renderings of the user's view to optimize annotation placement. Our method significantly improves over traditional camera-based view management in terms of annotation placement and legibility. Eye-perspective optimizations open up opportunities for further research on use cases relying on the user's true view through OST HMDs.",
keywords = "augmented reality, head-mounted displays, label placement, legibility, optical see-through, Cameras, Helmet mounted displays, Head-mounted-displays, Label placement, Legibility, Optical see-through, Optical see-through display, Optical see-through head-mounted displays, Perspective views, Real-world, User view, View management, Augmented reality",
author = "G. Emsenhuber and T. Langlotz and D. Kalkofen and J. Sutton and M. Tatzgern",
note = "Conference code: 188036 Export Date: 14 December 2023 Funding details: MFP-UOO2124 Funding details: {\"O}sterreichische Forschungsf{\"o}rderungsgesellschaft, FFG, 877104 Funding text 1: This work was supported by a grant from the Austrian Research Promotion Agency (grant no. 877104). Tobias Langlotz and Jonathan Sutton are supported by the Marsden Fund Council from Government funding (grant no. MFP-UOO2124) References: Baricevic, D., H{\"o}llerer, T., Sen, P., Turk, M., Userperspective augmented reality magic lens from gradients (2014) Acm Symposium on Virtual Reality Software and Technology, pp. 87-96. , https://doi.org/10.1145/2671015.2671027; Baricevic, D., H{\"o}llerer, T., Sen, P., Turk, M., User-Perspective AR Magic Lens from Gradient-Based IBR and Semi-Dense Stereo (2017) Ieee Transactions on Visualization and Computer Graphics, 23 (7), pp. 1838-1851. , https://doi.org/10.1109/TVCG.2016.2559483, 2017; Baricevic, D., Lee, C., Turk, M., H{\"o}llerer, T., Bowman, D.A., A hand-held AR magic lens with user-perspective rendering (2012) 2012 Ieee International Symposium on Mixed and Augmented Reality (ISMAR), pp. 197-206. , https://doi.org/10.1109/ISMAR.2012.6402557; Bell, B., Feiner, S., H{\"o}llerer, T., View management for virtual and augmented reality (2001) Proceedings of the 14th Annual Acm Symposium on User Interface Software and Technology, pp. 101-110. , https://doi.org/10.1145/502348.502363, ACM, Orlando, Florida; Augusto Borsoi, R., Holsbach Costa, G., On the Performance and Implementation of Parallax Free Video See-Through Displays (2018) Ieee Transactions on Visualization and Computer Graphics, 24 (6), pp. 2011-2022. , https://doi.org/10.1109/TVCG.2017.2705184, 2018; Buehler, C., Bosse, M., McMillan, L., Gortler, S., Cohen, M., Unstructured lumigraph rendering (2001) Proc. of the 28th Annual Conference on Computer Graphics and Interactive Techniques, Siggraph 2001, pp. 425-432. , https://doi.org/10.1145/383259.383309, 2001; Chaurasia, G., Nieuwoudt, A., Ichim, A., Szeliski, R., Sorkine-Hornung, A., Passthrough+: Real-Time Stereoscopic View Synthesis for Mobile Mixed Reality (2020) Proc. Acm Comput. Graph. Interact. Tech, 3 (1), p. 17. , https://doi.org/10.1145/3384540, 2020, Article 7, apr; Debernardis, S., Fiorentino, M., Gattullo, M., Monno, G., Emmanuele Uva, A., Text Readability in Head-Worn Displays: Color and Style Optimization in Video versus Optical See-Through Devices (2014) Ieee Transactions on Visualization and Computer Graphics, 20 (1), pp. 125-139. , https://doi.org/10.1109/TVCG.2013.86, 2014; Dudley, J.J., Jacques, J.T., Ola Kristensson, P., Crowdsourcing Design Guidance for Contextual Adaptation of Text Content in Augmented Reality (2021) Proceedings of the 2021 Chi Conference on Human Factors in Computing Systems (Yokohama, Japan) (CHI '21), p. 14. , https://doi.org/10.1145/3411764.3445493, Association for Computing Machinery, New York, NY, USA, Article 731; Emsenhuber, G., Domhardt, M., Langlotz, T., Kalkofen, D., Tatzgern, M., Towards Eye-Perspective Rendering for Optical See-Through Head-Mounted Displays (2022) 2022 Ieee Conference on Virtual Reality and 3D User Interfaces Abstracts and Workshops (VRW), pp. 640-641. , https://doi.org/10.1109/VRW55335.2022.00171; Erat, O., Hoell, M., Haubenwallner, K., Pirchheim, C., Schmalstieg, D., Real-Time View Planning for Unstructured Lumigraph Modeling (2019) Ieee Trans. on Visualization and Computer Graphics, 25 (11), pp. 3063-3072. , https://doi.org/10.1109/TVCG.2019.2932237, 2019; Fritz, C.O., Morris, P.E., Richler, J.J., Efect size estimates: Current use, calculations, and interpretation (2012) Journal of Experimental Psychology: General, 141 (1), pp. 2-18. , https://doi.org/10.1037/a0024338, 2012; Gabbard, J.L., Edward Swan, J., Hix, D., The Efects of Text Drawing Styles, Background Textures, and Natural Lighting on Text Legibility in Outdoor Augmented Reality (2006) Presence, 15 (1), pp. 16-32. , https://doi.org/10.1162/pres.2006.15.1.16, 2006; Gabbard, J.L., Swan, J.E., Hix, D., Kim, S., Fitch, G., Active Text Drawing Styles for Outdoor Augmented Reality: A User-Based Study and Design Implications (2007) Ieee Virtual Reality, pp. 35-42. , https://doi.org/10.1109/VR.2007.352461; Gattullo, M., Emmanuele Uva, A., Meccanica, D., Bari, P., Bari, P., Meccanica, D., Predicting Text Legibility over Textured Digital Backgrounds for a Monocular Optical See-Through Display (2015) Presence, 26 (1), pp. 1-15. , https://doi.org/10.1162/PRES, 2015; Gortler, S.J., Grzeszczuk, R., Szeliski, R., Cohen, M.F., The lumigraph (1996) Proceedings of the 23rd Annual Conference on Computer Graphics and Interactive Techniques, Siggraph 1996, pp. 43-54. , https://doi.org/10.1145/237170.237200, 1996; Grasset, R., Langlotz, T., Kalkofen, D., Tatzgern, M., Schmalstieg, D., Image-driven view management for augmented reality browsers (2012) 2012 Ieee Int. Symp. on Mixed and Augmented Reality (ISMAR), pp. 177-186. , https://doi.org/10.1109/ISMAR.2012.6402555; Gruber, L., Kalkofen, D., Schmalstieg, D., Color harmonization for Augmented Reality (2010) Ieee International Symposium on Mixed and Augmented Reality, pp. 227-228. , https://doi.org/10.1109/ISMAR.2010.5643580; Grubert, J., Itoh, Y., Moser, K., Edward Swan, J., A Survey of Calibration Methods for Optical See-Through Head-Mounted Displays (2018) Ieee Transactions on Visualization and Computer Graphics, 24 (9), pp. 2649-2662. , https://doi.org/10.1109/TVCG.2017.2754257, 2018; Henderson, S., Feiner, S., Exploring the benefts of augmented reality documentation for maintenance and repair (2011) Ieee Transactions on Visualization and Computer Graphics, 17 (10), pp. 1355-1368. , https://doi.org/10.1109/TVCG.2010.245, 2011; Hill, A., Schiefer, J., Wilson, J., Davidson, B., Gandy, M., MacIntyre, B., Virtual transparency: Introducing parallax view into video seethrough AR (2011) Ieee International Symposium on Mixed and Augmented Reality, pp. 239-240. , https://doi.org/10.1109/ISMAR.2011.6092395; David Hincapi{\'e}-Ramos, J., Ivanchuk, L., Sridharan, S.K., Irani, P.P., (2015) Ieee Transactions on Visualization and Computer Graphics, 21 (12), pp. 1336-1348. , https://doi.org/10.1109/TVCG.2015.2450745, 2015, SmartColor: Real-Time Color and Contrast Correction for Optical See-Through Head-Mounted Displays; Itoh, Y., Dzitsiuk, M., Amano, T., Klinker, G., Semi-Parametric Color Reproduction Method for Optical See-Through Head-Mounted Displays (2015) Ieee Trans. on Visualization and Computer Graphics, 21 (11), pp. 1269-1278. , https://doi.org/10.1109/TVCG.2015.2459892, 2015; Itoh, Y., Langlotz, T., Sutton, J., Plopski, A., Towards Indistinguishable Augmented Reality: A Survey on Optical See-through Head-Mounted Displays (2021) Acm Comput. Surv, 54 (6), p. 36. , https://doi.org/10.1145/3453157, 2021, Article 120, July; Langlotz, T., Cook, M., Regenbrecht, H., Real-time radiometric compensation for optical see-through head-mounted displays (2016) Ieee Transactions on Visualization and Computer Graphics, 22 (11), pp. 2385-2394. , https://doi.org/10.1109/TVCG.2016.2593781, 2016; Langlotz, T., Nguyen, T., Schmalstieg, D., Grasset, R., Next-Generation Augmented Reality Browsers: Rich, Seamless, and Adaptive (2014) Proc. Ieee, 102 (2), pp. 155-169. , https://doi.org/10.1109/JPROC.2013.2294255, 2014; Langlotz, T., Sutton, J., Zollmann, S., Itoh, Y., Regenbrecht, H., ChromaGlasses: Computational glasses for compensating colour blindness (2018) Conf. on Human Factors in Computing Systems, pp. 1-12. , https://doi.org/10.1145/3173574.3173964, 2018-April; Leykin, A., Tuceryan, M., Automatic determination of text readability over textured backgrounds for augmented reality systems (2004) Third Ieee and Acm International Symposium on Mixed and Augmented Reality, pp. 224-230. , https://doi.org/10.1109/ISMAR.2004.22; Lipson, L., Teed, Z., Deng, J., RAFT-Stereo: Multilevel Recurrent Field Transforms for Stereo Matching (2021) 2021 International Conference on 3D Vision (3DV), pp. 218-227. , https://doi.org/10.1109/3DV53792.2021.00032, IEEE Computer Society, Los Alamitos, CA, USA; Boesen Madsen, J., Tatzqern, M., Madsen, C.B., Schmalstieg, D., Kalkofen, D., Temporal Coherence Strategies for Augmented Reality Labeling (2016) Ieee Trans. on Vis. and Computer Graphics, 22 (4), pp. 1415-1423. , https://doi.org/10.1109/TVCG.2016.2518318, 2016; Matsuda, Y., Shibata, F., Kimura, A., Tamura, H., Poster: Creating a user-specifc perspective view for mobile mixed reality systems on smartphones (2013) Ieee Symposium on 3D User Interface, pp. 157-158. , https://doi.org/10.1109/3DUI.2013.6550226; Mildenhall, B., Srinivasan, P.P., Ortiz-Cayon, R., Khademi Kalantari, N., Ramamoorthi, R., Ng, R., Kar, A., Local Light Field Fusion: Practical View Synthesis with Prescriptive Sampling Guidelines (2019) Acm Trans. Graph, 38 (4), p. 14. , https://doi.org/10.1145/3306346.3322980, 2019, Article 29, jul; Miles, W.R., Ocular dominance in human adults (1930) Journal of General Psychology, 3 (3), pp. 412-430. , https://doi.org/10.1080/00221309.1930.9918218, 1930; Min Htike, H., Margrain, T.H., Lai, Y., Eslambolchilar, P., Augmented Reality Glasses as an Orientation and Mobility Aid for People with Low Vision: A Feasibility Study of Experiences and Requirements (2021) Proceedings of the 2021 Chi Conference on Human Factors in Computing Systems (Yokohama, Japan) (CHI '21), p. 15. , https://doi.org/10.1145/3411764.3445327, New York, NY, USA, Article 729; Mohr, P., Tatzgern, M., Grubert, J., Schmalstieg, D., Kalkofen, D., Adaptive user perspective rendering for Handheld Augmented Reality (2017) 2017 Ieee Symposium on 3D User Interfaces, 3DUI 2017 -Proceedings, , https://doi.org/10.1109/3DUI.2017.7893336; Mori, S., Erat, O., Broll, W., Saito, H., Schmalstieg, D., Kalkofen, D., InpaintFusion: Incremental RGB-D Inpainting for 3D Scenes (2020) Ieee Transactions on Visualization and Computer Graphics, 26 (10), pp. 2994-3007. , https://doi.org/10.1109/TVCG.2020.3003768, 2020; Mori, S., Schmalstieg, D., Kalkofen, D., Good Keyframes to Inpaint (2022) Ieee Transactions on Visualization and Computer Graphics, p. 1. , https://doi.org/10.1109/TVCG.2022.3176958, 2022; Nguyen, C., DiVerdi, S., Hertzmann, A., Liu, F., Depth Confict Reduction for Stereo VR Video Interfaces (2018) Proc. of the 2018 Chi Conference on Human Factors in Computing Systems (CHI '18), pp. 1-9. , https://doi.org/10.1145/3173574.3173638, New York, NY, USA; Nuernberger, B., Ofek, E., Benko, H., Wilson, A.D., SnapToReality: Aligning Augmented Reality to the Real World (2016) Proc. of the 2016 Chi Conf. on Human Factors in Computing Systems, pp. 1233-1244. , https://doi.org/10.1145/2858036.2858250, New York, New York, USA; Orlosky, J., Kiyokawa, K., Takemura, H., Dynamic text management for see-through wearable and heads-up display systems (2013) Proc. of the 2013 International Conference on Intelligent User Interfaces (IUI '13), pp. 363-370. , https://doi.org/10.1145/2449396.2449443; Paas, F.G., Training strategies for attaining transfer of problem solving skills in statistics : A cognitive-load approach (1992) Journal of Educational Psychology, 84 (4), pp. 429-434. , https://doi.org/10.1037/0022-0663.84.4.429, 1992; Peterson, S.D., Axholt, M., Ellis, S.R., Comparing Disparity Based Label Segregation in Augmented and Virtual Reality (2008) Proc. of the 2008 Acm Symposium on Virtual Reality Software and Technology (VRST '08), pp. 285-286. , https://doi.org/10.1145/1450579.1450655; Peterson, S.D., Axholt, M., Ellis, S.R., Objective and subjective assessment of stereoscopically separated labels in augmented reality (2009) Computers \& Graphics, 33 (1), pp. 23-33. , https://doi.org/10.1016/j.cag.2008.11.006, 2009; Rosten, E., Reitmayr, G., Drummond, T., Real-Time Video Annotations for Augmented Reality (2005) Advances in Visual Computing, pp. 294-302. , George Bebis, Richard Boyle, Darko Koracin, and Bahram Parvin (Eds.), Springer Berlin Heidelberg, Berlin, Heidelberg; Samini, A., Lundin Palmerius, J., A User Study on Touch Interaction for User-Perspective Rendering in Hand-Held Video See-Through Augmented Reality (2016) Augmented Reality, Virtual Reality, and Computer Graphics, pp. 304-317. , https://doi.org/10.1007/978-3-31940651-0, Springer International Publishing; Samini, A., Lundin Palmerius, K., A perspective geometry approach to user-perspective rendering in hand-held video see-through augmented reality (2014) Acm Symposium on Virtual Reality Software and Technology, pp. 207-208. , https://doi.org/10.1145/2671015.2671127; Sch{\"o}ps, T., Oswald, M.R., Speciale, P., Yang, S., Pollefeys, M., Real-Time View Correction for Mobile Devices (2017) Ieee Transactions on Visualization and Computer Graphics, 23 (11), pp. 2455-2462. , https://doi.org/10.1109/TVCG.2017.2734578, 2017; Sidenmark, L., Kiefer, N., Gellersen, H., Subtitles in interactive virtual reality: Using gaze to address depth conficts (2019) Workshop on Emerging Novel Input Devices and Interaction Techniques; Kirshnamachari Sridharan, S., David Hincapi{\'e}-Ramos, J., Flatla, D.R., Irani, P., Color Correction for Optical See-through Displays Using Display Color Profles (2013) Proceedings of the 19th Acm Symposium on Virtual Reality Software and Technology (Singapore) (VRST '13), pp. 231-240. , https://doi.org/10.1145/2503713.2503716, Association for Computing Machinery, New York, NY, USA; Sutton, J., Langlotz, T., Plopski, A., Seeing Colours: Addressing Colour Vision Defciency with Vision Augmentations Using Computational Glasses (2022) Acm Trans. Comput.-Hum. Interact, 29 (3), p. 53. , https://doi.org/10.1145/3486899, 2022, Article 26, jan; Tang, O., Biocca, M., Comparative efectiveness of augmented reality in object assembly (2003) Proceedings of the Sigchi Conference on Human Factors in Computing Systems, pp. 73-80. , https://doi.org/10.1145/642611.642626, ACM, New York, NY, USA; Tatzgern, M., Kalkofen, D., Grasset, R., Schmalstieg, D., Hedgehog labeling: View management techniques for external labels in 3D space (2014) 2014 Ieee Virtual Reality (VR), pp. 27-32. , https://doi.org/10.1109/VR.2014.6802046; Teather, R.J., Stuerzlinger, W., Factors Afecting Mouse-Based 3D Selection in Desktop VR Systems (2015) Proceedings of the 3rd Acm Symposium on Spatial User Interaction (Los Angeles, California, USA) (SUI '15), pp. 10-19. , https://doi.org/10.1145/2788940.2788946; Tomioka, M., Ikeda, S., Sato, K., Approximated userperspective rendering in tablet-based augmented reality (2013) 2013 Ieee International Symposium on Mixed and Augmented Reality, Ismar 2013, pp. 21-28. , https://doi.org/10.1109/ISMAR.2013.6671760, 2013; Tuceryan, M., Navab, N., Single point active alignment method (SPAAM) for optical see-through HMD calibration for AR (2000) Proceedings -IEEE and Acm International Symposium on Augmented Reality, Isar 2000, 11 (3), pp. 149-158. , https://doi.org/10.1109/ISAR.2000.880938, 2000; Klen, C.P., Paul, C., Jason, A., Evaluating dual-view perceptual issues in handheld augmented reality : Device vs. user perspective rendering (2013) International Conference on Multimodal Interaction, pp. 381-388. , https://doi.org/10.1145/2522848.2522885; Copic Pucihar, K., Coulton, P., Alexander, J., The use of surrounding visual context in handheld AR (2014) Conference on Human Factors in Computing Systems, pp. 197-206. , https://doi.org/10.1145/2556288.2557125; Xiao, L., Nouri, S., Hegland, J., Garcia Garcia, A., Lanman, D., (2022) NeuralPassthrough: Learned Real-Time View Synthesis for Vr, , https://doi.org/10.48550/ARXIV.2207.02186, 2022; Yamaguchi, M., Mori, S., Mohr, P., Tatzgern, M., Stanescu, A., Saito, H., Kalkofen, D., Video-Annotated Augmented Reality Assembly Tutorials (2020) Acm User Interface Software and Technology (UIST '20), pp. 1010-1022. , https://doi.org/10.1145/3379337.3415819; Zhou, Q., Fitzmaurice, G., Anderson, F., In-Depth Mouse: Integrating Desktop Mouse into Virtual Reality (2022) Proceedings of the 2022 Chi Conference on Human Factors in Computing Systems (New Orleans, LA, USA) (CHI '22), p. 17. , https://doi.org/10.1145/3491102.3501884, Association for Computing Machinery, New York, NY, USA, Article 354; Zuf, S., Brambilla, C., Beretta, G., Scala, P., Human Computer Interaction: Legibility and Contrast (2007) 14th International Conference on Image Analysis and Processing (ICIAP, pp. 241-246. , https://doi.org/10.1109/ICIAP.2007.4362786, 2007; 2023 CHI Conference on Human Factors in Computing Systems, CHI 2023, CHI 2023 ; Conference date: 23-04-2023 Through 28-04-2023",
year = "2023",
month = apr,
day = "19",
doi = "10.1145/3544548.3581059",
language = "English",
isbn = "978-1-4503-9421-5",
series = "Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems",
publisher = "Association for Computing Machinery",
booktitle = "Conference on Human Factors in Computing Systems - Proceedings",
address = "United States",
url = "https://chi2023.acm.org/",
}