@InProceedings{memik:patterns, author = {Gokhan Memik and Mahmut Kandemir and Alok Choudhary}, title = {Exploiting Inter-File Access Patterns Using Multi-Collective {I/O}}, booktitle = {Proceedings of the USENIX FAST '02 Conference on File and Storage Technologies}, year = {2002}, month = {January}, pages = {245--258}, publisher = {USENIX Association}, address = {Monterey, CA}, URL = {http://www.usenix.org/publications/library/proceedings/fast02/memik.html}, keywords = {file systems, pario-bib}, abstract = {This paper introduces a new concept called Multi-Collective I/O (MCIO) that extends conventional collective I/O to optimize I/O accesses to multiple arrays simultaneously. In this approach, as in collective I/O, multiple processors co-ordinate to perform I/O on behalf of each other if doing so improves overall I/O time. However, unlike collective I/O, MCIO considers multiple arrays simultaneously; that is, it has a more global view of the overall I/O behavior exhibited by application. This paper shows that determining optimal MCIO access pattern is an NP-complete problem, and proposes two different heuristics for the access pattern detection problem (also called the assignment problem). Both of the heuristics have been implemented within a runtime library, and tested using a large-scale scientific application. Our preliminary results show that MCIO out-performs collective I/O by as much as 87{PCT}. Our runtime library-based implementation can be used by users as well as optimizing compilers. Based on our results, we recom-mend future library designers for I/O-intensive applications to include MCIO in their suite of optimizations.} }