Talks | Parallel Computing

@misc{22:eage,

title = {Hybrid Workflows For Large-Scale Scientific Applications},

author = {Iacopo Colonnelli and Marco Aldinucci},

url = {https://datacloud.di.unito.it/index.php/s/GScPS5LCPdt6Yoo},

year  = {2022},

date = {2022-09-01},

address = {Milano, Italy},

abstract = {Large-scale scientific applications are facing an irreversible transition from monolithic, high-performance oriented codes to modular and polyglot deployments of specialised (micro-)services. The reasons behind this transition are many: coupling of standard solvers with Deep Learning techniques, offloading of data analysis and visualisation to Cloud, and the advent of specialised hardware accelerators. Topology-aware Workflow Management Systems (WMSs) play a crucial role. In particular, topology-awareness allows an explicit mapping of workflow steps onto heterogeneous locations, allowing automated executions on top of hybrid architectures (e.g., cloud+HPC or classical+quantum). Plus, topology-aware WMSs can offer non-functional requirements OOTB, e.g. components’ life-cycle orchestration, secure and efficient data transfers, fault tolerance, and cross-cluster execution of urgent workloads. Augmenting interactive Jupyter Notebooks with distributed workflow capabilities allows domain experts to prototype and scale applications using the same technological stack, while relying on a feature-rich and user-friendly web interface. This abstract will showcase how these general methodologies can be applied to a typical geoscience simulation pipeline based on the Full Wavefront Inversion (FWI) technique. In particular, a prototypical Jupyter Notebook will be executed interactively on Cloud. Preliminary data analyses and post-processing will be executed locally, while the computationally demanding optimisation loop will be scheduled on a remote HPC cluster.},

howpublished = {6th EAGE High Performance Computing Workshop},

keywords = {across, eupex, jupyter-workflow, textarossa},

pubstate = {published},

tppubtype = {misc}

}

Iacopo Colonnelli, Dario Tranchitella

Dossier: multi-tenant distributed Jupyter Notebooks Miscellaneous

DoK Talks 141, 2022, (Invited talk).

Abstract | Links | BibTeX | Tags: across, deephealth, hpc4ai, jupyter-workflow

@misc{22:data-on-kubernetes,

title = {Dossier: multi-tenant distributed Jupyter Notebooks},

author = {Iacopo Colonnelli and Dario Tranchitella},

url = {https://datacloud.di.unito.it/index.php/s/RNqTGmTqWS66qHT},

year  = {2022},

date = {2022-07-01},

address = {Virtual event},

abstract = {When providing data analysis as a service, one must tackle several problems. Data privacy and protection by design are crucial when working on sensitive data. Performance and scalability are fundamental for compute-intensive workloads, e.g. training Deep Neural Networks. User-friendly interfaces and fast prototyping tools are essential to allow domain experts to experiment with new techniques. Portability and reproducibility are necessary to assess the actual value of results. Kubernetes is the best platform to provide reliable, elastic, and maintainable services. However, Kubernetes alone is not enough to achieve large-scale multi-tenant reproducible data analysis. OOTB support for multi-tenancy is too rough, with only two levels of segregation (i.e. the single namespace or the entire cluster). Offloading computation to off-cluster resources is non-trivial and requires the user's manual configuration. Also, Jupyter Notebooks per se cannot provide much scalability (they execute locally and sequentially) and reproducibility (users can run cells in any order and any number of times). The Dossier platform allows system administrators to manage multi-tenant distributed Jupyter Notebooks at the cluster level in the Kubernetes way, i.e. through CRDs. Namespaces are aggregated in Tenants, and all security and accountability aspects are managed at that level. Each Notebook spawns into a user-dedicated namespace, subject to all Tenant-level constraints. Users can rely on provisioned resources, either in-cluster worker nodes or external resources like HPC facilities. Plus, they can plug their computing nodes in a BYOD fashion. Notebooks are interpreted as distributed workflows, where each cell is a task that one can offload to a different location in charge of its execution.},

howpublished = {DoK Talks 141},

note = {Invited talk},

keywords = {across, deephealth, hpc4ai, jupyter-workflow},

pubstate = {published},

tppubtype = {misc}

}