Talks | Parallel Computing

@misc{24:icolonne:cwl4hpccern,

title = {CWL in the HPC Ecosystem},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/PRmqdwWHt6P2PH7},

year  = {2024},

date = {2024-04-01},

address = {CERN, Meyrin, Switzerland},

howpublished = {Workshop on workflow languages for HEP analysis},

keywords = {across, eupex, icsc, space, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{24:icsc:spoke1:eni,

title = {Cross-Platform Full Waveform Inversion},

author = {Alberto Mulone},

url = {https://datacloud.di.unito.it/index.php/s/M3HkxA5wsBPS5ro},

year  = {2024},

date = {2024-02-01},

address = {Torino, Italy},

abstract = {Presentation about the Innovation Grant in collaboration with ENI},

howpublished = {ICSC - Spoke 1 meeting},

keywords = {icsc, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{23:sc:WCIBoF,

title = {Orchestrating Multi-Domain Workflows: The ACROSS Approach},

author = {Alberto Scionti and Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/rJXcDBK4mLmS8yz},

year  = {2023},

date = {2023-11-01},

address = {Denver, CO, Usa},

howpublished = {Workflows Community: Modern Workflows for Continuum and Cross-Facility Computing},

keywords = {across, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{23:AssembleaHPC-KTT,

title = {ACROSS: HPC Big Data Artificial Intelligence Cross Stack Platform Towards Exascale},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/aK7es8BgFeWorjD},

year  = {2023},

date = {2023-10-01},

address = {Pisa, Italy},

howpublished = {LN HPC-KTT Assemblea Nazionale 2023},

keywords = {across, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{23:HaMMonProject,

title = {Università degli Studi di Torino: Alpha parallel research group},

author = {Iacopo Colonnelli and Doriana Medić and Barbara Cantalupo and Marco Aldinucci},

url = {https://datacloud.di.unito.it/index.php/s/cmgy9BZ3nwCR2QJ},

year  = {2023},

date = {2023-10-01},

address = {Bologna, Italy},

howpublished = {HaMMon Kick-Off meeting},

keywords = {icsc, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{23:ITADATABestPhDThesis,

title = {Workflow models for heterogeneous distributed systems},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/6RqcaJ4djqFNDC8},

year  = {2023},

date = {2023-09-01},

address = {Napoli, Italy},

howpublished = {2nd Italian Conference on Big Data and Data Science (ITADATA 2023)},

note = {Best PhD Thesis Award},

keywords = {jupyter-workflow, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{23:mulone:wide:vcp,

title = {Porting the Variant Calling Pipeline for NGS data in cloud-HPC environment},

author = {Alberto Mulone and Sherine Awad and Davide Chiarugi and Marco Aldinucci},

editor = {Hossain Shahriar and Yuuichi Teranishi and Alfredo Cuzzocrea and Moushumi Sharmin and Dave Towey and A. K. M. Jahangir Alam Majumder and Hiroki Kashiwazaki and Ji-Jiang Yang and Michiharu Takemoto and Nazmus Sakib and Ryohei Banno and Sheikh Iqbal Ahamed},

url = {https://datacloud.di.unito.it/index.php/s/zNLj3LCZNsNxHwy},

doi = {10.1109/COMPSAC57700.2023.00288},

year  = {2023},

date = {2023-06-01},

booktitle = {47th IEEE Annual Computers, Software, and Applications Conference, COMPSAC 2023},

pages = {1858–1863},

publisher = {IEEE},

address = {Torino, Italy},

abstract = {In recent years we have understood the importance of analyzing and sequencing human genetic variation. A relevant aspect that emerged from the Covid-19 pandemic was the need to obtain results very quickly; this involved using High-Performance Computing (HPC) environments to execute the Next Generation Sequencing (NGS) pipeline. However, HPC is not always the most suitable environment for the entire execution of a pipeline, especially when it involves many heterogeneous tools. The ability to execute parts of the pipeline on different environments can lead to higher performance but also cheaper executions. This work shows the design and optimization process that led us to a state-of-the-art Variant Calling hybrid workflow based on the StreamFlow Workflow Management System (WfMS). We also compare StreamFlow with Snakemake, an established WfMS targeting HPC facilities, observing comparable performance on single environments and satisfactory improvements with a hybrid cloud-HPC configuration.},

howpublished = {47th IEEE Annual Computers, Software, and Applications Conference, COMPSAC 2023},

keywords = {across, icsc, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{23:SA2Micro,

title = {Workflows and the Common Workflow Language (CWL)},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/NHWWzMMaQgAsA52},

year  = {2023},

date = {2023-05-01},

address = {Torino, Italy},

howpublished = {OSA2Micro: An Open Science Approach to Microbiology data integration},

note = {Invited talk},

keywords = {streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{23:FL3WorkingDay,

title = {UNITO tools presentation},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/fgHbnLDQSFtcwLd},

year  = {2023},

date = {2023-05-01},

address = {Bologna, Italy},

howpublished = {CN HPC Flagship 3 Working Day},

keywords = {jupyter-workflow, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{23:HBPSummit,

title = {Standardised Workflows at EBRAINS},

author = {Sofia Karvounari and Eleni Mathioulaki and Michael R. Crusoe and Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/K5YQKTsX9N7NLT8},

year  = {2023},

date = {2023-03-01},

address = {Marseille, France},

abstract = {A hands-on training offer for Standardised Workflows in EBRAINS. A short presentation will be used as an introduction, while the main hands-on session will provide information about Writing and Executing Standardised Workflows. TC will give some guidelines, so attendees can experiment with writing CWL tools and workflows and then they will be given access to VM to execute these workflows. The Workflows Dashboard will be also presented during the same session, offering to the attendees the opportunity to understand the different functionalities, use it with TC support and provide useful comments.},

howpublished = {Human Brain Project Summit 2023},

note = {Invited talk},

keywords = {across, eupex, space, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{23:CWLConference,

title = {CWL for HPC: are we there yet?},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/CMCd5LiZeXsxwEg},

year  = {2023},

date = {2023-03-01},

address = {Heidelberg, Germany},

abstract = {Modern HPC applications are becoming so heterogeneous and complex that a modular approach to their design, deployment and orchestration is now necessary. This talk explores the benefits of using a vendor-agnostic workflow language (CWL) coupled with a hybrid workflow management system (StreamFlow) in the HPC ecosystem. Also, it will examine the requirements needed to model HPC applications effectively, the CWL’s readiness to meet such requirements, and the proposals made to improve the language where needed. Four real use cases will drive the discussion: the ACROSS Project (G.A. n. 955648), where CWL is the primary interface to model three HPC workflows, and the EUPEX Project (G.A. n. 101033975), where StreamFlow will be used for the rapid prototyping of a seismic engineering HPC application for a Modular Supercomputing Architecture (MSA) system.},

howpublished = {2023 CWL Conference},

note = {Invited talk},

keywords = {across, eupex, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{22:healthycloud-workshop,

title = {StreamFlow},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/Taz8qtzmkmn9ffT},

year  = {2022},

date = {2022-07-01},

address = {Virtual event},

howpublished = {2nd HealthyCloud Workshop: Analysis of existing orchestration mechanisms for distributed computational analyses},

note = {Invited talk},

keywords = {across, deephealth, eupex, streamflow, textarossa},

pubstate = {published},

tppubtype = {misc}

}

@misc{22:elixir-streamflow,

title = {StreamFlow: a topology-aware WMS},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/Z9GsKnRCxmBdMd3},

year  = {2022},

date = {2022-06-01},

address = {Virtual event},

howpublished = {ELIXIR Cloud, Data & AAI Bi-weekly Technical Calls},

note = {Invited talk},

keywords = {across, deephealth, eupex, streamflow, textarossa},

pubstate = {published},

tppubtype = {misc}

}

@misc{22:eupex-streamflow,

title = {StreamFlow: A framework for hybrid workflows},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/NjKEySP7HfrCQHZ},

year  = {2022},

date = {2022-04-01},

address = {Virtual event},

howpublished = {EUPEX WP5 bi-weekly meeting},

keywords = {eupex, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{22:jotb22,

title = {OpenDeepHealth: Crafting a Deep Learning Platform as a Service with Kubernetes},

author = {Iacopo Colonnelli and Dario Tranchitella},

url = {https://datacloud.di.unito.it/index.php/s/n6J7STNnwdyqtET},

year  = {2022},

date = {2022-04-01},

address = {Malaga, Spain},

howpublished = {J on The Beach 2022},

keywords = {across, deephealth, hpc4ai, jupyter-workflow, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{22:jotb22-workshop,

title = {Distributed workflows with Jupyter},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/om89q55S6ePf2Ji},

year  = {2022},

date = {2022-04-01},

address = {Malaga, Spain},

howpublished = {J on The Beach 2022},

note = {Workshop},

keywords = {across, deephealth, jupyter-workflow, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{22:across-streamflow,

title = {StreamFlow: A framework for hybrid workflows},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/FXFTKtQSRf6anMX},

year  = {2022},

date = {2022-02-01},

address = {Virtual event},

howpublished = {ACROSS WP4 meeting},

keywords = {across, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{21:across-streamflow,

title = {StreamFlow: A framework for hybrid workflows},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/yrGYJL6CyNywF8a},

year  = {2021},

date = {2021-10-01},

address = {Virtual event},

howpublished = {ACROSS WP4 meeting},

keywords = {across, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{21:eni:streamflow,

title = {The modernization of HPC applications for the cloud era},

author = {Marco Aldinucci},

year  = {2021},

date = {2021-09-01},

address = {Virtual event},

abstract = {Workflows are among the most commonly used tools in a variety of execution environments. Many of them target a specific environment; few of them make it possible to execute an entire workflow in different environments, e.g., clouds, supercomputers, and both of them. We present a novel approach to workflow execution, called StreamFlow, that complements the workflow graph with the declarative description of potentially complex execution environments (such as Kubernetes and SLURM), making it possible to execute onto multiple sites not sharing a common data space. Streamflow clearly distinguishes it from many other workflow management systems because it decouples the data dependencies from the deployment of (containerized) workflow steps. Streamflow also leverages CAPIO (Cross-Application Programmable I/O) to move data from one step to another efficiently. CAPIO captures the POSIX file system and streams it in parallel and in-memory to the workflow's next step, possibly enabling in-transit data filtering.},

howpublished = {Fifth EAGE Workshop on High Performance Computing for Upstream},

note = {Keynote talk},

keywords = {across, admire, deephealth, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{21:hlpp:streamflow,

title = {From skeletons to workflows in the cloud-edge era},

author = {Marco Aldinucci},

url = {https://datacloud.di.unito.it/index.php/s/RyRPjNBse5PKnab},

year  = {2021},

date = {2021-07-01},

address = {Virtual event},

abstract = {Workflows are among the most commonly used tools in a variety of execution environments. Many of them target a specific environment; few of them make it possible to execute an entire workflow in different environments, e.g. Kubernetes and batch clusters. We present a novel approach to workflow execution, called StreamFlow, that complements the workflow graph with the declarative description of potentially complex execution environments and that makes it possible to execute multiple sites not sharing a common data space. StreamFlow supports both task and data parallelism and enables the reproducible and scalable execution of workflows, such as AI pipelines, in hybrid cloud-HPC environments. As a running example, we use the novel ``universal COVID-19 pipeline'' that explore the whole optimisation space of the training of different DNNs to classify COVID-19 lung lesions.},

howpublished = {14th Intl. Symposium on High-Level Programming and Applications (HLPP)},

note = {Keynote talk},

keywords = {across, admire, deephealth, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{21:gtc:clairecovid,

title = {The Universal Cloud-HPC Pipeline for the AI-Assisted Explainable Diagnosis of COVID-19 Pneumonia},

author = {Marco Aldinucci and Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/AkQLbPpEEtDzbbm},

year  = {2021},

date = {2021-04-01},

address = {Virtual event},

abstract = {We'll present a methodology to run DNN pipelines on hybrid cloud+HPC infrastructure. We'll also define a "universal pipeline" for medical images. The pipeline can reproduce all state-of-the-art DNNs to diagnose COVID-19 pneumonia, which appeared in the literature during the first Italian lockdown and following months. We can run all of them (across cloud+HPC platforms) and compare their performance in terms of sensitivity and specificity to set a baseline to evaluate future progress in the automated diagnosis of COVID-19. Also, the pipeline makes existing DNNs explainable by way of adversarial training. The pipeline is easily portable and can run across different infrastructures, adapting the performance-urgency trade-off. The methodology builds onto two novel software programs: the streamflow workflow system and the AI-sandbox concept (parallel container with user-space encrypted file system). We reach over 92% accuracy in diagnosing COVID pneumonia.},

howpublished = {NVidia GTC'21},

note = {Invited talk},

keywords = {deephealth, hpc4ai, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{21:CWLMiniConference,

title = {StreamFlow: cross breeding cloud with HPC},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/Le9gg4PfjRxBwXD},

year  = {2021},

date = {2021-02-01},

address = {Virtual event},

abstract = {Workflows are among the most commonly used tools in a variety of execution environments. Many of them target a specific environment; few of them make it possible to execute an entire workflow in different environments, e.g. Kubernetes and batch clusters. We present a novel approach to workflow execution, called StreamFlow, that complements the workflow graph with the declarative description of potentially complex execution environments, and that makes it possible the execution onto multiple sites not sharing a common data space.},

howpublished = {2021 CWL Mini Conference},

note = {Invited talk},

keywords = {deephealth, streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{20:HPCEuropa3TAM,

title = {StreamFlow: cross breeding cloud with HPC},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/qPHHrSNxk8QXJDw},

year  = {2020},

date = {2020-10-01},

address = {Virtual event},

abstract = {Workflows are among the most commonly used tools in a variety of execution environments. Many of them target a specific environment; few of them make it possible to execute an entire workflow in different environments, e.g. Kubernetes and batch clusters. We present a novel approach to workflow execution, called StreamFlow, that complements the workflow graph with the declarative description of potentially complex execution environments, and that makes it possible the execution onto multiple sites not sharing a common data space. StreamFlow is then exemplified on a novel bioinformatics pipeline for single-cell transcriptomic data analysis workflow.},

howpublished = {HPC-Europa3 2nd Transnational Access Meeting (TAM)},

note = {Invited talk},

keywords = {streamflow},

pubstate = {published},

tppubtype = {misc}

}

@misc{19:GarrWorkshop,

title = {StreamFlow: un approccio dichiarativo a workflow e pipeline di micro-servizi},

author = {Iacopo Colonnelli},

url = {https://datacloud.di.unito.it/index.php/s/kZqyiQnBEQNdXJe},

year  = {2019},

date = {2019-10-01},

address = {Roma, Italy},

abstract = {Negli ultimi anni, gli approcci orientati ai container si sono dimostrati particolarmente efficaci nel garantire portabilità e riproducibilità dei workflow scientifici. Tuttavia, con il continuo aumento del volume di dati a disposizione e la crescente complessità delle procedure di analisi in ogni campo della ricerca, anche i requisiti di performance e riusabilità si fanno via via sempre più essenziali. L'obiettivo principale di StreamFlow è quello di fornire un nuovo paradigma, totalmente dichiarativo, per la descrizione e l'accelerazione di workflow scientifici in ambienti distribuiti. La peculiarità di StreamFlow risiede nel fatto che l'ambiente di esecuzione è interamente descritto in termini di servizi (container), connessioni tra essi e fattori di replica. Inoltre, ogni task del workflow è esplicitamente mappato sulla tipologia di servizio richiesta. Questo permette un maggior controllo sull'utilizzo delle risorse e politiche di scheduling più precise, a vantaggio delle performance. I principali vantaggi di un approccio dichiarativo sono invece la più facile comprensione ed estensione dei modelli esistenti, a vantaggio della riusabilitià.},

howpublished = {Workshop GARR 2019},

keywords = {streamflow},

pubstate = {published},

tppubtype = {misc}

}