Objave

@article{PR_Fairness2024,

title = {Fairness in Face Presentation Attack Detection},

author = {Meiling Fang and Wufei Yang and Arjan Kuijper and Vitomir S̆truc and Naser Damer},

url = {https://www.sciencedirect.com/science/article/pii/S0031320323007008?dgcid=coauthor},

year  = {2024},

date = {2024-03-01},

urldate = {2024-03-01},

journal = {Pattern Recognition},

volume = {147 },

issue = {110002},

pages = {1-14},

abstract = {Face recognition (FR) algorithms have been proven to exhibit discriminatory behaviors against certain demographic and non-demographic groups, raising ethical and legal concerns regarding their deployment in real-world scenarios. Despite the growing number of fairness studies in FR, the fairness of face presentation attack detection (PAD) has been overlooked, mainly due to the lack of appropriately annotated data. To avoid and mitigate the potential negative impact of such behavior, it is essential to assess the fairness in face PAD and develop fair PAD models. To enable fairness analysis in face PAD, we present a Combined Attribute Annotated PAD Dataset (CAAD-PAD), offering seven human-annotated attribute labels. Then, we comprehensively analyze the fairness of PAD and its relation to the nature of the training data and the Operational Decision Threshold Assignment (ODTA) through a set of face PAD solutions. Additionally, we propose a novel metric, the Accuracy Balanced Fairness (ABF), that jointly represents both the PAD fairness and the absolute PAD performance. The experimental results pointed out that female and faces with occluding features (e.g. eyeglasses, beard, etc.) are relatively less protected than male and non-occlusion groups by all PAD solutions. To alleviate this observed unfairness, we propose a plug-and-play data augmentation method, FairSWAP, to disrupt the identity/semantic information and encourage models to mine the attack clues. The extensive experimental results indicate that FairSWAP leads to better-performing and fairer face PADs in 10 out of 12 investigated cases.},

keywords = {biometrics, computer vision, face analysis, face PAD, face recognition, fairness, pad, presentation attack detection},

pubstate = {published},

tppubtype = {article}

}

@article{MaskFaceGAN,

title = {MaskFaceGAN: High Resolution Face Editing With Masked GAN Latent Code Optimization},

author = {Martin Pernuš and Vitomir Štruc and Simon Dobrišek},

url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=10299582

https://lmi.fe.uni-lj.si/wp-content/uploads/2023/02/MaskFaceGAN_compressed.pdf

https://arxiv.org/pdf/2103.11135.pdf},

doi = {10.1109/TIP.2023.3326675},

issn = {1941-0042},

year  = {2023},

date = {2023-10-27},

urldate = {2023-01-02},

journal = {IEEE Transactions on Image Processing},

abstract = {Face editing represents a popular research topic within the computer vision and image processing communities. While significant progress has been made recently in this area, existing solutions: ( i ) are still largely focused on low-resolution images, ( ii ) often generate editing results with visual artefacts, or ( iii ) lack fine-grained control over the editing procedure and alter multiple (entangled) attributes simultaneously, when trying to generate the desired facial semantics. In this paper, we aim to address these issues through a novel editing approach, called MaskFaceGAN that focuses on local attribute editing. The proposed approach is based on an optimization procedure that directly optimizes the latent code of a pre-trained (state-of-the-art) Generative Adversarial Network (i.e., StyleGAN2) with respect to several constraints that ensure: ( i ) preservation of relevant image content, ( ii ) generation of the targeted facial attributes, and ( iii ) spatially–selective treatment of local image regions. The constraints are enforced with the help of an (differentiable) attribute classifier and face parser that provide the necessary reference information for the optimization procedure. MaskFaceGAN is evaluated in extensive experiments on the FRGC, SiblingsDB-HQf, and XM2VTS datasets and in comparison with several state-of-the-art techniques from the literature. Our experimental results show that the proposed approach is able to edit face images with respect to several local facial attributes with unprecedented image quality and at high-resolutions (1024×1024), while exhibiting considerably less problems with attribute entanglement than competing solutions. The source code is publicly available from: https://github.com/MartinPernus/MaskFaceGAN.},

keywords = {CNN, computer vision, deep learning, face editing, face image processing, GAN, GAN inversion, generative models, StyleGAN},

pubstate = {published},

tppubtype = {article}

}

@article{PrivacProberRot,

title = {PrivacyProber: Assessment and Detection of Soft–Biometric Privacy–Enhancing Techniques},

author = {Peter Rot and Klemen Grm and Peter Peer and Vitomir Štruc},

url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=10264192},

doi = {10.1109/TDSC.2023.3319500},

isbn = {1545-5971},

year  = {2023},

date = {2023-09-23},

journal = {IEEE Transactions on Dependable and Secure Computing},

pages = {1-18},

abstract = {Soft–biometric privacy–enhancing techniques represent machine learning methods that aim to: (i) mitigate privacy concerns associated with face recognition technology by suppressing selected soft–biometric attributes in facial images (e.g., gender, age, ethnicity) and (ii) make unsolicited extraction of sensitive personal information infeasible. Because such techniques are increasingly used in real–world applications, it is imperative to understand to what extent the privacy enhancement can be inverted and how much attribute information can be recovered from privacy–enhanced images. While these aspects are critical, they have not been investigated in the literature so far. In this paper, we, therefore, study the robustness of several state–of–the–art soft–biometric privacy–enhancing techniques to attribute recovery attempts. We propose PrivacyProber, a high–level framework for restoring soft–biometric information from privacy–enhanced facial images, and apply it for attribute recovery in comprehensive experiments on three public face datasets, i.e., LFW, MUCT and Adience. Our experiments show that the proposed framework is able to restore a considerable amount of suppressed information, regardless of the privacy–enhancing technique used (e.g., adversarial perturbations, conditional synthesis, etc.), but also that there are significant differences between the considered privacy models. These results point to the need for novel mechanisms that can improve the robustness of existing privacy–enhancing techniques and secure them against potential adversaries trying to restore suppressed information. Additionally, we demonstrate that PrivacyProber can also be used to detect privacy–enhancement in facial images (under black–box assumptions) with high accuracy. Specifically, we show that a detection procedure can be developed around the proposed framework that is learning free and, therefore, generalizes well across different data characteristics and privacy–enhancing techniques.},

keywords = {biometrics, face, privacy, privacy enhancement, privacy protection, privacy-enhancing techniques, soft biometric privacy},

pubstate = {published},

tppubtype = {article}

}

@article{VitekSaud2023,

title = {IPAD: Iterative Pruning with Activation Deviation for Sclera Biometrics},

author = {Matej Vitek and Matic Bizjak and Peter Peer and Vitomir Štruc},

url = {https://lmi.fe.uni-lj.si/wp-content/uploads/2023/07/PublishedVersion.pdf},

doi = {https://doi.org/10.1016/j.jksuci.2023.101630},

year  = {2023},

date = {2023-07-10},

journal = {Journal of King Saud University - Computer and Information Sciences},

volume = {35},

number = {8},

pages = {1-21},

abstract = {The sclera has recently been gaining attention as a biometric modality due to its various desirable characteristics. A key step in any type of ocular biometric recognition, including sclera recognition, is the segmentation of the relevant part(s) of the eye. However, the high computational complexity of the (deep) segmentation models used in this task can limit their applicability on resource-constrained devices such as smartphones or head-mounted displays. As these devices are a common desired target for such biometric systems, lightweight solutions for ocular segmentation are critically needed. To address this issue, this paper introduces IPAD (Iterative Pruning with Activation Deviation), a novel method for developing lightweight convolutional networks, that is based on model pruning. IPAD uses a novel filter-activation-based criterion (ADC) to determine low-importance filters and employs an iterative model pruning procedure to derive the final lightweight model. To evaluate the proposed pruning procedure, we conduct extensive experiments with two diverse segmentation models, over four publicly available datasets (SBVPI, SLD, SMD and MOBIUS), in four distinct problem configurations and in comparison to state-of-the-art methods from the literature. The results of the experiments show that the proposed filter-importance criterion outperforms the standard L1 and L2 approaches from the literature. Furthermore, the results also suggest that: 1) the pruned models are able to retain (or even improve on) the performance of the unpruned originals, as long as they are not over-pruned, with RITnet and U-Net at 50% of their original FLOPs reaching up to 4% and 7% higher IoU values than their unpruned versions, respectively, 2) smaller models require more careful pruning, as the pruning process can hurt the model’s generalization capabilities, and 3) the novel criterion most convincingly outperforms the classic approaches when sufficient training data is available, implying that the abundance of data leads to more robust activation-based importance computation.},

keywords = {biometrics, CNN, deep learning, model compression, pruning, sclera, sclera segmentation},

pubstate = {published},

tppubtype = {article}

}

@article{AccessMartin2023,

title = {ChildNet: Structural Kinship Face Synthesis Model With Appearance Control Mechanisms},

author = {Martin Pernuš and Mansi Bhatnagar and Badr Samad and Divyanshu Singh and Peter Peer and Vitomir Štruc and Simon Dobrišek},

url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=10126110},

doi = {10.1109/ACCESS.2023.3276877},

issn = {2169-3536},

year  = {2023},

date = {2023-05-17},

journal = {IEEE Access},

pages = {1-22},

abstract = {Kinship face synthesis is an increasingly popular topic within the computer vision community, particularly the task of predicting the child appearance using parental images. Previous work has been limited in terms of model capacity and inadequate training data, which is comprised of low-resolution and tightly cropped images, leading to lower synthesis quality. In this paper, we propose ChildNet,  a method for kinship face synthesis that leverages the facial image generation capabilities of a state-of-the-art Generative Adversarial Network (GAN), and resolves the aforementioned problems. ChildNet is designed within the GAN latent space and is able to predict a child appearance that bears high resemblance to real parents’ children. To ensure fine-grained control, we propose an age and gender manipulation module that allows precise manipulation of the child synthesis result. ChildNet is capable of generating multiple child images per parent pair input, while providing a way to control the image generation variability. Additionally, we introduce a mechanism to control the dominant parent image. Finally, to facilitate the task of kinship face synthesis, we introduce a new kinship dataset, called Next of Kin. This dataset contains 3690 high-resolution face images with a diverse range of ethnicities and ages. We evaluate ChildNet in comprehensive experiments against three competing kinship face synthesis models, using two kinship datasets. The experiments demonstrate the superior performance of ChildNet in terms of identity similarity, while exhibiting high perceptual image quality. The source code for the model is publicly available at: https://github.com/MartinPernus/ChildNet.},

keywords = {artificial intelligence, CNN, deep learning, face generation, face synthesis, GAN, GAN inversion, kinship, kinship synthesis, StyleGAN2},

pubstate = {published},

tppubtype = {article}

}

@article{FadiIVCSynthetic,

title = {Synthetic data for face recognition: Current state and future prospects},

author = {Fadi Boutros and Vitomir Štruc and Julian Fierrez and Naser Damer},

url = {https://www.sciencedirect.com/science/article/pii/S0262885623000628},

doi = {https://doi.org/10.1016/j.imavis.2023.104688},

year  = {2023},

date = {2023-05-15},

urldate = {2023-05-15},

journal = {Image and Vision Computing},

number = {104688},

abstract = {Over the past years, deep learning capabilities and the availability of large-scale training datasets advanced rapidly, leading to breakthroughs in face recognition accuracy. However, these technologies are foreseen to face a major challenge in the next years due to the legal and ethical concerns about using authentic biometric data in AI model training and evaluation along with increasingly utilizing data-hungry state-of-the-art deep learning models. With the recent advances in deep generative models and their success in generating realistic and high-resolution synthetic image data, privacy-friendly synthetic data has been recently proposed as an alternative to privacy-sensitive authentic data to overcome the challenges of using authentic data in face recognition development. This work aims at providing a clear and structured picture of the use-cases taxonomy of synthetic face data in face recognition along with the recent emerging advances of face recognition models developed on the bases of synthetic data. We also discuss the challenges facing the use of synthetic data in face recognition development and several future prospects of synthetic data in the domain of face recognition.},

keywords = {biometrics, CNN, diffusion, face recognition, generative models, survey, synthetic data},

pubstate = {published},

tppubtype = {article}

}

@article{Grabner_TSG,

title = {A global modeling framework for load forecasting in distribution networks},

author = {Miha Grabner and Yi Wang and Qingsong Wen and Boštjan Blažič and Vitomir Štruc},

url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=10092804},

doi = {10.1109/TSG.2023.3264525},

issn = {1949-3061},

year  = {2023},

date = {2023-04-05},

journal = {IEEE Transactions on Smart Grid},

abstract = {With the increasing numbers of smart meter installations, scalable and efficient load forecasting techniques are critically needed to ensure sustainable situation awareness within the distribution networks. Distribution networks include a large amount of different loads at various aggregation levels, such as individual consumers, low-voltage feeders, and transformer stations. It is impractical to develop individual (or so-called local) forecasting models for each load separately. Additionally, such local models also (i) (largely) ignore the strong dependencies between different loads that might be present due to their spatial proximity and the characteristics of the distribution network, (ii) require historical data for each load to be able to make forecasts, and (iii) are incapable of adjusting to changes in the load behavior without retraining. To address these issues, we propose a global modeling framework for load forecasting in distribution networks that, unlike its local competitors, relies on a single global model to generate forecasts for a large number of loads. The global nature of the framework, significantly reduces the computational burden typically required when training multiple local forecasting models, efficiently exploits the cross-series information shared among different loads, and facilitates forecasts even when historical data for a load is missing or the behavior of a load evolves over time. To further improve on the performance of the proposed framework, an unsupervised localization mechanism and optimal ensemble construction strategy are also proposed to localize/personalize the global forecasting model to different load characteristics. Our experimental results show that the proposed framework outperforms naive benchmarks by more than 25% (in terms of Mean Absolute Error) on real-world dataset while exhibiting highly desirable characteristics when compared to the local models that are predominantly used in the literature. All source code and data are made publicly available to enable reproducibility: https://github.com/mihagrabner/GlobalModelingFramework},

keywords = {deep learning, global modeling, load forecasting, prediction, smart grid, time series analysis, time series forecasting},

pubstate = {published},

tppubtype = {article}

}

@article{MedenDeID2023,

title = {Face deidentification with controllable privacy protection},

author = {Blaž Meden and Manfred Gonzalez-Hernandez and Peter Peer and Vitomir Štruc},

url = {https://reader.elsevier.com/reader/sd/pii/S0262885623000525?token=BC1E21411C50118E666720B002A89C9EB3DB4CFEEB5EB18D7BD7B0613085030A96621C8364583BFE7BAE025BE3646096&originRegion=eu-west-1&originCreation=20230516115322},

doi = {https://doi.org/10.1016/j.imavis.2023.104678},

year  = {2023},

date = {2023-04-01},

journal = {Image and Vision Computing},

volume = {134},

number = {104678},

pages = {1-19},

abstract = {Privacy protection has become a crucial concern in today’s digital age. Particularly sensitive here are facial images, which typically not only reveal a person’s identity, but also other sensitive personal information. To address this problem, various face deidentification techniques have been presented in the literature. These techniques try to remove or obscure personal information from facial images while still preserving their usefulness for further analysis. While a considerable amount of work has been proposed on face deidentification, most state-of-theart solutions still suffer from various drawbacks, and (a) deidentify only a narrow facial area, leaving potentially important contextual information unprotected, (b) modify facial images to such degrees, that image naturalness and facial diversity is suffering in the deidentify images, (c) offer no flexibility in the level of privacy protection ensured, leading to suboptimal deployment in various applications, and (d) often offer an unsatisfactory tradeoff between the ability to obscure identity information, quality and naturalness of the deidentified images, and sufficient utility preservation. In this paper, we address these shortcomings with a novel controllable face deidentification technique that balances image quality, identity protection, and data utility for further analysis. The proposed approach utilizes a powerful generative model (StyleGAN2), multiple auxiliary classification models, and carefully designed constraints to guide the deidentification process. The approach is validated across four diverse datasets (CelebA-HQ, RaFD, XM2VTS, AffectNet) and in comparison to 7 state-of-the-art competitors. The results of the experiments demonstrate that the proposed solution leads to: (a) a considerable level of identity protection, (b) valuable preservation of data utility, (c) sufficient diversity among the deidentified faces, and (d) encouraging overall performance.},

keywords = {CNN, deep learning, deidentification, face recognition, GAN, GAN inversion, privacy, privacy protection, StyleGAN2},

pubstate = {published},

tppubtype = {article}

}

@article{TIFS_Sclera2022,

title = {Exploring Bias in Sclera Segmentation Models: A Group Evaluation Approach},

author = {Matej Vitek and Abhijit Das and Diego Rafael Lucio and Luiz Antonio Zanlorensi Jr. and David Menotti and Jalil Nourmohammadi Khiarak and Mohsen Akbari Shahpar and Meysam Asgari-Chenaghlu and Farhang Jaryani and Juan E. Tapia and Andres Valenzuela and Caiyong Wang and Yunlong Wang and Zhaofeng He and Zhenan Sun and Fadi Boutros and Naser Damer and Jonas Henry Grebe and Arjan Kuijper and Kiran Raja and Gourav Gupta and Georgios Zampoukis and Lazaros Tsochatzidis and Ioannis Pratikakis and S. V. Aruna Kumar and B. S. Harish and Umapada Pal and Peter Peer and Vitomir Štruc},

url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=9926136},

doi = {10.1109/TIFS.2022.3216468},

issn = {1556-6013},

year  = {2023},

date = {2023-01-18},

urldate = {2022-10-18},

journal = {IEEE Transactions on Information Forensics and Security},

volume = {18},

pages = {190-205},

abstract = {Bias and fairness of biometric algorithms have been key topics of research in recent years, mainly due to the societal, legal and ethical implications of potentially unfair decisions made by automated decision-making models. A considerable amount of work has been done on this topic across different biometric modalities, aiming at better understanding the main sources of algorithmic bias or devising mitigation measures. In this work, we contribute to these efforts and present the first study investigating bias and fairness of sclera segmentation models. Although sclera segmentation techniques represent a key component of sclera-based biometric systems with a considerable impact on the overall recognition performance, the presence of different types of biases in sclera segmentation methods is still underexplored. To address this limitation, we describe the results of a group evaluation effort (involving seven research groups), organized to explore the performance of recent sclera segmentation models within a common experimental framework and study performance differences (and bias), originating from various demographic as well as environmental factors. Using five diverse datasets, we analyze seven independently developed sclera segmentation models in different experimental configurations. The results of our experiments suggest that there are significant differences in the overall segmentation performance across the seven models and that among the considered factors, ethnicity appears to be the biggest cause of bias. Additionally, we observe that training with representative and balanced data does not necessarily lead to less biased results. Finally, we find that in general there appears to be a negative correlation between the amount of bias observed (due to eye color, ethnicity and acquisition device) and the overall segmentation performance, suggesting that advances in the field of semantic segmentation may also help with mitigating bias.},

keywords = {bias, biometrics, fairness, group evaluation, ocular, sclera, sclera segmentation, segmentation},

pubstate = {published},

tppubtype = {article}

}

@article{IVC2023,

title = {A Survey on Computer Vision based Human Analysis in the COVID-19 Era},

author = {Fevziye Irem Eyiokur and Alperen Kantarci and Mustafa Ekrem Erakin and Naser Damer and Ferda Ofli and Muhammad Imran and Janez Križaj and Albert Ali Salah and Alexander Waibel and Vitomir Štruc and Hazim K. Ekenel },

url = {https://lmi.fe.uni-lj.si/wp-content/uploads/2023/01/FG4COVID19_PAPER_compressed.pdf

https://authors.elsevier.com/a/1gKOyxnVK7RBS},

doi = {https://doi.org/10.1016/j.imavis.2022.104610},

year  = {2023},

date = {2023-01-01},

journal = {Image and Vision Computing},

volume = {130},

number = {104610},

pages = {1-19},

abstract = {The emergence of COVID-19 has had a global and profound impact, not only on society as a whole, but also on the lives of individuals. Various prevention measures were introduced around the world to limit the transmission of the disease, including 

face masks, mandates for social distancing and regular disinfection in public spaces, and the use of screening applications. These developments also triggered the need for novel and improved computer vision techniques capable of (i) providing support to the prevention measures through an automated analysis of visual data, on the one hand, and (ii) facilitating normal operation of existing vision-based services, such as biometric authentication schemes, on the other. Especially important here, are computer vision techniques that focus on the analysis of people and faces in visual data and have been affected the most by the partial occlusions introduced by the mandates for facial masks. 

Such computer vision based human analysis techniques include face and face-mask detection approaches, face recognition techniques, crowd counting solutions, age and expression estimation procedures, models for detecting face-hand interactions and many others, and have seen considerable attention over recent years. The goal of this survey is to provide an introduction to the problems induced by COVID-19 into such research and to present a comprehensive review of the work done in the computer vision based human analysis field. Particular attention is paid to the impact of facial masks on the performance of various methods and recent solutions to mitigate this problem. Additionally, a detailed review of existing datasets useful for the development and evaluation of methods for COVID-19 related applications is also provided. Finally, to help advance the field further, a discussion on the main open challenges and future research direction is given at the end of the survey. This work is intended to have a broad appeal and be useful not only for computer vision researchers but also the general public.},

keywords = {COVID-19, face, face alignment, face analysis, face image processing, face image quality assessment, face landmarking, face recognition, face verification, human analysis, masked face analysis},

pubstate = {published},

tppubtype = {article}

}

@article{UhljiIETZiga,

title = {Efficient ear alignment using a two-stack hourglass network},

author = {Anja Hrovatič and Peter Peer and Vitomir Štruc and Žiga Emeršič},

url = {https://ietresearch.onlinelibrary.wiley.com/doi/epdf/10.1049/bme2.12109},

doi = {10.1049/bme2.12109},

issn = {2047-4938},

year  = {2023},

date = {2023-01-01},

journal = {IET Biometrics },

pages = {1-14},

abstract = {Ear images have been shown to be a reliable modality for biometric recognition with desirable characteristics, such as high universality, distinctiveness, measurability and permanence. While a considerable amount of research has been directed towards ear recognition techniques, the problem of ear alignment is still under-explored in the open literature. Nonetheless, accurate alignment of ear images, especially in unconstrained acquisition scenarios, where the ear appearance is expected to vary widely due to pose and view point variations, is critical for the performance of all downstream tasks, including ear recognition. Here, the authors address this problem and present a framework for ear alignment that relies on a two-step procedure: (i) automatic landmark detection and (ii) fiducial point alignment. For the first (landmark detection) step, the authors implement and train a Two-Stack Hourglass model (2-SHGNet) capable of accurately predicting 55 landmarks on diverse ear images captured in uncontrolled conditions. For the second (alignment) step, the authors use the Random Sample Consensus (RANSAC) algorithm to align the estimated landmark/fiducial points with a pre-defined ear shape (i.e. a collection of average ear landmark positions). The authors evaluate the proposed framework in comprehensive experiments on the AWEx and ITWE datasets and show that the 2-SHGNet model leads to more accurate landmark predictions than competing state-of-the-art models from the literature. Furthermore, the authors also demonstrate that the alignment step significantly improves recognition accuracy with ear images from unconstrained environments compared to unaligned imagery.},

keywords = {biometrics, CNN, deep learning, ear, ear alignment, ear recognition},

pubstate = {published},

tppubtype = {article}

}

@article{TextEmotionESWA,

title = {DHF-Net: A hierarchical feature interactive fusion network for dialogue emotion recognition},

author = {Chenquan Gan and Yucheng Yang and Qingyi Zhub and Deepak Kumar Jain and Vitomir Struc},

url = {https://www.sciencedirect.com/science/article/pii/S0957417422016025?via%3Dihub},

doi = {https://doi.org/10.1016/j.eswa.2022.118525},

year  = {2022},

date = {2022-12-30},

urldate = {2022-08-01},

journal = {Expert Systems with Applications},

volume = {210},

abstract = {To balance the trade-off between contextual information and fine-grained information in identifying specific emotions during a dialogue and combine the interaction of hierarchical feature related information, this paper proposes a hierarchical feature interactive fusion network (named DHF-Net), which not only can retain the integrity of the context sequence information but also can extract more fine-grained information. To obtain a deep semantic information, DHF-Net processes the task of recognizing dialogue emotion and dialogue act/intent separately, and then learns the cross-impact of two tasks through collaborative attention. Also, a bidirectional gate recurrent unit (Bi-GRU) connected hybrid convolutional neural network (CNN) group method is designed, by which the sequence information is smoothly sent to the multi-level local information layers for feature exaction. Experimental results show that, on two open session datasets, the performance of DHF-Net is improved by 1.8% and 1.2%, respectively.},

keywords = {attention, CNN, deep learning, dialogue, emotion recognition, fusion, fusion network, nlp, semantics, text, text processing},

pubstate = {published},

tppubtype = {article}

}

@article{TomasevicSensors,

title = {Reconstructing Superquadrics from Intensity and Color Images},

author = {Darian Tomašecić and Peter Peer and Franc Solina and Aleš Jaklič and Vitomir Štruc},

url = {https://www.mdpi.com/1424-8220/22/14/5332/pdf?version=1658380987},

doi = {https://doi.org/10.3390/s22145332},

year  = {2022},

date = {2022-07-16},

journal = {Sensors},

volume = {22},

number = {5332},

issue = {4},

abstract = {The task of reconstructing 3D scenes based on visual data represents a longstanding problem in computer vision. Common reconstruction approaches rely on the use of multiple volumetric primitives to describe complex objects. Superquadrics (a class of volumetric primitives) have shown great promise due to their ability to describe various shapes with only a few parameters. Recent research has shown that deep learning methods can be used to accurately reconstruct random superquadrics from both 3D point cloud data and simple depth images. In this paper, we extended these reconstruction methods to intensity and color images. Specifically, we used a dedicated convolutional neural network (CNN) model to reconstruct a single superquadric from the given input image. We analyzed the results in a qualitative and quantitative manner, by visualizing reconstructed superquadrics as well as observing error and accuracy distributions of predictions. We showed that a CNN model designed around a simple ResNet backbone can be used to accurately reconstruct superquadrics from images containing one object, but only if one of the spatial parameters is fixed or if it can be determined from other image characteristics, e.g., shadows. Furthermore, we experimented with images of increasing complexity, for example, by adding textures, and observed that the results degraded only slightly. In addition, we show that our model outperforms the current state-of-the-art method on the studied task. Our final result is a highly accurate superquadric reconstruction model, which can also reconstruct superquadrics from real images of simple objects, without additional training.},

keywords = {arrs, CNN, depth data, depth estimation, depth sensing, intensity images, superquadric, superquadrics},

pubstate = {published},

tppubtype = {article}

}

@article{TBIOM_2022,

title = {An Attack on Feature Level-based Facial Soft-biometric Privacy Enhancement},

author = {Daile Osorio-Roig and Christian Rathgeb and Pawel Drozdowski and Philipp Terhörst and Vitomir Štruc and Christoph Busch},

url = {https://arxiv.org/pdf/2111.12405.pdf},

year  = {2022},

date = {2022-05-02},

urldate = {2022-05-02},

journal = {IEEE Transactions on Biometrics, Identity and Behavior (TBIOM)},

volume = {4},

issue = {2},

pages = {263-275},

abstract = {In the recent past, different researchers have proposed novel privacy-enhancing face recognition systems designed to conceal soft-biometric information at feature level. These works have reported impressive results, but usually do not consider specific attacks in their analysis of privacy protection. In most cases, the privacy protection capabilities of these schemes are tested through simple machine learning-based classifiers and visualisations of dimensionality reduction tools. In this work, we introduce an attack on feature level-based facial soft–biometric privacy-enhancement techniques. The attack is based on two observations: (1) to achieve high recognition accuracy, certain similarities between facial representations have to be retained in their privacy-enhanced versions; (2) highly similar facial representations usually originate from face images with similar soft-biometric attributes. Based on these observations, the proposed attack compares a privacy-enhanced face representation against a set of privacy-enhanced face representations with known soft-biometric attributes. Subsequently, the best obtained similarity scores are analysed to infer the unknown soft-biometric attributes of the attacked privacy-enhanced face representation. That is, the attack only requires a relatively small database of arbitrary face images and the privacy-enhancing face recognition algorithm as a black-box. In the experiments, the attack is applied to two representative approaches which have previously been reported to reliably conceal the gender in privacy-enhanced face representations. It is shown that the presented attack is able to circumvent the privacy enhancement to a considerable degree and is able to correctly classify gender with an accuracy of up to approximately 90% for both of the analysed privacy-enhancing face recognition systems. Future works on privacy-enhancing face recognition are encouraged to include the proposed attack in evaluations on privacy protection.},

keywords = {attack, face recognition, privacy, privacy enhancement, privacy protection, privacy-enhancing techniques, soft biometric privacy},

pubstate = {published},

tppubtype = {article}

}

@article{KrizajSensors2022,

title = {Making the most of single sensor information : a novel fusion approach for 3D face recognition using region covariance descriptors and Gaussian mixture models},

author = {Janez Križaj and Simon Dobrišek and Vitomir Štruc},

url = {https://www.mdpi.com/1424-8220/22/6/2388},

doi = {10.3390/s22062388},

year  = {2022},

date = {2022-03-01},

journal = {Sensors},

number = {2388},

issue = {6},

pages = {1-26},

abstract = {Most commercially successful face recognition systems combine information from multiple sensors (2D and 3D, visible light and infrared, etc.) to achieve reliable recognition in various environments. When only a single sensor is available, the robustness as well as efficacy of the recognition process suffer. In this paper, we focus on face recognition using images captured by a single 3D sensor and propose a method based on the use of region covariance matrixes and Gaussian mixture models (GMMs). All steps of the proposed framework are automated, and no metadata, such as pre-annotated eye, nose, or mouth positions is required, while only a very simple clustering-based face detection is performed. The framework computes a set of region covariance descriptors from local regions of different face image representations and then uses the unscented transform to derive low-dimensional feature vectors, which are finally modeled by GMMs. In the last step, a support vector machine classification scheme is used to make a decision about the identity of the input 3D facial image. The proposed framework has several desirable characteristics, such as an inherent mechanism for data fusion/integration (through the region covariance matrixes), the ability to explore facial images at different levels of locality, and the ability to integrate a domain-specific prior knowledge into the modeling procedure. Several normalization techniques are incorporated into the proposed framework to further improve performance. Extensive experiments are performed on three prominent databases (FRGC v2, CASIA, and UMB-DB) yielding competitive results.},

keywords = {3d face, biometrics, face, face analysis, face images, face recognition},

pubstate = {published},

tppubtype = {article}

}

@article{Stoimchev2022,

title = {Learning to Combine Local and Global Image Information for Contactless Palmprint Recognition},

author = {Marjan Stoimchev and Marija Ivanovska and Vitomir Štruc},

url = {https://lmi.fe.uni-lj.si/wp-content/uploads/2022/03/sensors-22-00073_reduced.pdf},

doi = {https://doi.org/10.3390/s22010073},

year  = {2022},

date = {2022-01-01},

journal = {Sensors},

volume = {22},

number = {1},

pages = {1-26},

abstract = {In the past few years, there has been a leap from traditional palmprint recognition methodologies, which use handcrafted features, to deep-learning approaches that are able to automatically learn feature representations from the input data. However, the information that is extracted from such deep-learning models typically corresponds to the global image appearance, where only the most discriminative cues from the input image are considered. This characteristic is especially problematic when data is acquired in unconstrained settings, as in the case of contactless palmprint recognition systems, where visual artifacts caused by elastic deformations of the palmar surface are typically present in spatially local parts of the captured images. In this study we address the problem of elastic deformations by introducing a new approach to contactless palmprint recognition based on a novel CNN model, designed as a two-path architecture, where one path processes the input in a holistic manner, while the second path extracts local information from smaller image patches sampled from the input image. As elastic deformations can be assumed to most significantly affect the global appearance, while having a lesser impact on spatially local image areas, the local processing path addresses the issues related to elastic deformations thereby supplementing the information from the global processing path. The model is trained with a learning objective that combines the Additive Angular Margin (ArcFace) Loss and the well-known center loss. By using the proposed model design, the discriminative power of the learned image representation is significantly enhanced compared to standard holistic models, which, as we show in the experimental section, leads to state-of-the-art performance for contactless palmprint recognition. Our approach is tested on two publicly available contactless palmprint datasets—namely, IITD and CASIA—and is demonstrated to perform favorably against state-of-the-art methods from the literature. The source code for the proposed model is made publicly available.},

keywords = {biometrics; computer vision; deep learning; palmprints},

pubstate = {published},

tppubtype = {article}

}

@article{ContexedNet_Emersic_2021,

title = {ContexedNet : Context-Aware Ear Detection in Unconstrained Settings},

author = {Žiga Emeršič and Diego Sušanj and Blaž Meden and Peter Peer and Vitomir Štruc},

editor = {ContexedNet : Context-Aware Ear Detection in Unconstrained Settings},

url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=9583244},

issn = {2169-3536},

year  = {2021},

date = {2021-10-20},

urldate = {2021-10-20},

journal = {IEEE Access},

pages = {1--17},

abstract = {Ear detection represents one of the key components of contemporary ear recognition systems. While significant progress has been made in the area of ear detection over recent years, most of the improvements are direct results of advances in the field of visual object detection. Only a limited number of techniques presented in the literature are domain--specific and designed explicitly with ear detection in mind. In this paper, we aim to address this gap and present a novel detection approach that does not rely only on general ear (object) appearance, but also exploits contextual information, i.e., face--part locations, to ensure accurate and robust ear detection with images captured in a wide variety of imaging conditions. The proposed approach is based on a Context--aware Ear Detection Network (ContexedNet) and poses ear detection as a semantic image segmentation problem. ContexedNet consists of two processing paths: 1) a context--provider that extracts probability maps corresponding to the locations of facial parts from the input image, and 2) a dedicated ear segmentation model that integrates the computed probability maps into a context--aware segmentation-based ear detection procedure. ContexedNet is evaluated in rigorous experiments on the AWE and UBEAR datasets and shown to ensure competitive performance when evaluated against state--of--the--art ear detection models from the literature. Additionally, because the proposed contextualization is model agnostic, it can also be utilized with other ear detection techniques to improve performance.},

keywords = {biometrics, contextual information, deep leraning, ear detection, ear recognition, ear segmentation, neural networks, segmentation},

pubstate = {published},

tppubtype = {article}

}

@article{TIFS_PrivacySurveyb,

title = {Privacy-Enhancing Face Biometrics: A Comprehensive Survey},

author = {Blaz Meden, Peter Rot, Philipp Terhorst, Naser Damer, Arjan Kuijper, Walter J. Scheirer, Arun Ross, Peter Peer, Vitomir Struc},

url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=9481149

https://lmi.fe.uni-lj.si/en/visual_privacy_of_faces__a_survey_preprint-compressed/},

doi = {10.1109/TIFS.2021.3096024},

year  = {2021},

date = {2021-07-12},

journal = {IEEE Transactions on Information Forensics and Security},

volume = {16},

pages = {4147-4183},

abstract = {Biometric recognition technology has made significant advances over the last decade and is now used across a number of services and applications. However, this widespread deployment has also resulted in privacy concerns and evolving societal expectations about the appropriate use of the technology. For example, the ability to automatically extract age, gender, race, and health cues from biometric data has heightened concerns about privacy leakage. Face recognition technology, in particular, has been in the spotlight, and is now seen by many as posing a considerable risk to personal privacy. In response to these and similar concerns, researchers have intensified efforts towards developing techniques and computational models capable of ensuring privacy to individuals, while still facilitating the utility of face recognition technology in several application scenarios.  These efforts have resulted in a multitude of privacy--enhancing techniques that aim at addressing privacy risks originating from biometric systems and providing technological solutions for legislative requirements set forth in privacy laws and regulations, such as GDPR. The goal of this overview paper is to provide a comprehensive introduction into privacy--related research in the area of biometrics and review existing work on textit{Biometric Privacy--Enhancing Techniques} (B--PETs) applied to face biometrics. To make this work useful for as wide of an audience as possible,  several key topics are covered as well, including evaluation strategies used with B--PETs, existing datasets, relevant standards, and regulations and critical open issues that will have to be addressed in the future. },

keywords = {biometrics, deidentification, face analysis, face deidentification, face recognition, face verification, FaceGEN, privacy, privacy protection, privacy-enhancing techniques, soft biometric privacy},

pubstate = {published},

tppubtype = {article}

}

@article{CrowdCountingPevec,

title = {Benchmarking Crowd-Counting Techniques across Image Characteristics},

author = {Klemen Pevec and Klemen Grm and Vitomir Štruc},

url = {https://lmi.fe.uni-lj.si/wp-content/uploads/2022/10/Pevec.pdf

https://ev.fe.uni-lj.si/5-2021/Pevec.pdf},

year  = {2021},

date = {2021-05-01},

journal = {Elektorethniski Vestnik},

volume = {88},

issue = {5},

pages = {227-235},

abstract = {Crowd--counting is a longstanding computer vision used in estimating the crowd sizes for security purposes at public protests in streets, public gatherings, for collecting crowd statistics at airports, malls, concerts, conferences, and other similar venues, and for monitoring people and crowds during public health crises (such as the one caused by COVID-19). Recently, the performance of automated methods for crowd--counting from single images has improved particularly due to the introduction of deep learning techniques and large labelled training datasets. However, the robustness of these methods to varying imaging conditions, such as weather, image perspective, and large variations in the crowd size has not been studied in-depth in the open literature. To address this gap, a systematic study on the robustness of four recently developed crowd--counting methods is performed in this paper to evaluate their performance with respect to variable (real-life) imaging scenarios that include different event types, weather conditions, image sources and crowd sizes. It is shown that the performance of the tested techniques is degraded in unclear weather conditions (i.e., fog, rain, snow) and also on images taken from large distances by drones. On the opposite, clear weather conditions, crowd--counting methods can provide accurate and usable results.},

keywords = {CNN, crowd counting, drones, image characteristics, model comparison, neural networks},

pubstate = {published},

tppubtype = {article}

}

@article{Batagelj2021,

title = {How to correctly detect face-masks for COVID-19 from visual information?},

author = {Borut Batagelj and Peter Peer and Vitomir Štruc and Simon Dobrišek},

url = {https://www.mdpi.com/2076-3417/11/5/2070/pdf},

doi = {10.3390/app11052070},

isbn = {2076-3417},

year  = {2021},

date = {2021-03-01},

urldate = {2021-03-01},

journal = {Applied sciences},

volume = {11},

number = {5},

pages = {1-24},

abstract = {The new Coronavirus disease (COVID-19) has seriously affected the world. By the end of November 2020, the global number of new coronavirus cases had already exceeded 60 million and the number of deaths 1,410,378 according to information from the World Health Organization (WHO). To limit the spread of the disease, mandatory face-mask rules are now becoming common in public settings around the world. Additionally, many public service providers require customers to wear face-masks in accordance with predefined rules (e.g., covering both mouth and nose) when using public services. These developments inspired research into automatic (computer-vision-based) techniques for face-mask detection that can help monitor public behavior and contribute towards constraining the COVID-19 pandemic. Although existing research in this area resulted in efficient techniques for face-mask detection, these usually operate under the assumption that modern face detectors provide perfect detection performance (even for masked faces) and that the main goal of the techniques is to detect the presence of face-masks only. In this study, we revisit these common assumptions and explore the following research questions: (i) How well do existing face detectors perform with masked-face images? (ii) Is it possible to detect a proper (regulation-compliant) placement of facial masks? and (iii) How useful are existing face-mask detection techniques for monitoring applications during the COVID-19 pandemic? To answer these and related questions we conduct a comprehensive experimental evaluation of several recent face detectors for their performance with masked-face images. Furthermore, we investigate the usefulness of multiple off-the-shelf deep-learning models for recognizing correct face-mask placement. Finally, we design a complete pipeline for recognizing whether face-masks are worn correctly or not and compare the performance of the pipeline with standard face-mask detection models from the literature. To facilitate the study, we compile a large dataset of facial images from the publicly available MAFA and Wider Face datasets and annotate it with compliant and non-compliant labels. The annotation dataset, called Face-Mask-Label Dataset (FMLD), is made publicly available to the research community.},

keywords = {computer vision, COVID-19, deep learning, detection, face, mask detection, recognition},

pubstate = {published},

tppubtype = {article}

}

@article{Oblak2021,

title = {Learning to predict superquadric parameters from depth images with explicit and implicit supervision},

author = {Tim Oblak and Jaka Šircelj and Vitomir Struc and Peter Peer and Franc Solina and Aleš Jaklic

},

url = {https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=9274424},

doi = {10.1109/ACCESS.2020.3041584},

issn = {2169-3536},

year  = {2021},

date = {2021-01-01},

journal = {IEEE Access},

pages = {1-16},

abstract = {Reconstruction of 3D space from visual data has always been a significant challenge in

the field of computer vision. A popular approach to address this problem can be found in the form of

bottom-up reconstruction techniques which try to model complex 3D scenes through a constellation of

volumetric primitives. Such techniques are inspired by the current understanding of the human visual

system and are, therefore, strongly related to the way humans process visual information, as suggested

by recent visual neuroscience literature. While advances have been made in recent years in the area of

3D reconstruction, the problem remains challenging due to the many possible ways of representing 3D

data, the ambiguity of determining the shape and general position in 3D space and the difficulty to train

efficient models for the prediction of volumetric primitives. In this paper, we address these challenges and

present a novel solution for recovering volumetric primitives from depth images. Specifically, we focus on

the recovery of superquadrics, a special type of parametric models able to describe a wide array of 3D

shapes using only a few parameters. We present a new learning objective that relies on the superquadric

(inside-outside) function and develop two learning strategies for training convolutional neural networks

(CNN) capable of predicting superquadric parameters. The first uses explicit supervision and penalizes the

difference between the predicted and reference superquadric parameters. The second strategy uses implicit

supervision and penalizes differences between the input depth images and depth images rendered from

the predicted parameters. CNN predictors for superquadric parameters are trained with both strategies and

evaluated on a large dataset of synthetic and real-world depth images. Experimental results show that both

strategies compare favourably to the existing state-of-the-art and result in high quality 3D reconstructions

of the modelled scenes at a much shorter processing time.},

keywords = {3d, computer vision, depth images, differential renderer, recovery, superquadric},

pubstate = {published},

tppubtype = {article}

}

@article{DBLP:journals/corr/abs-2103-11135,

title = {High Resolution Face Editing with Masked GAN Latent Code Optimization},

author = {Martin Pernus and Vitomir Struc and Simon Dobrisek},

url = {https://arxiv.org/abs/2103.11135},

year  = {2021},

date = {2021-01-01},

urldate = {2021-01-01},

journal = {CoRR},

volume = {abs/2103.11135},

keywords = {},

pubstate = {published},

tppubtype = {article}

}