BibTeX for papers by David Kotz; for complete/updated list see https://www.cs.dartmouth.edu/~kotz/research/papers.html @TechReport{vengroff:holesome-tr, author = {Darren Erik Vengroff and David Kotz}, title = {{A Holesome File System}}, institution = {Dartmouth Computer Science}, year = 2004, month = {May}, number = {TR2004-497}, copyright = {the authors}, URL = {https://www.cs.dartmouth.edu/~kotz/research/vengroff-holesome-tr/index.html}, note = {Originally written in July 1995; released May 2004.}, abstract = {We present a novel approach to fully dynamic management of physical disk blocks in Unix file systems. By adding a single system call, \emph{zero()}, to an existing file system, we permit applications to create \emph{holes}, that is, regions of files to which no physical disk blocks are allocated, far more flexibly than previously possible. \emph{zero} can create holes in the middle of existing files. \par Using \emph{zero()}, it is possible to efficiently implement applications including a variety of databases and I/O-efficient computation systems on top of the Unix file system. \emph{zero()} can also be used to implement an efficient file-system-based paging mechanism. In some I/O-efficient computations, the availability of \emph{zero()} effectively doubles disk capacity by allowing blocks of temporary files to be reallocated to new files as they are read. \par Experiments on a Linux \emph{ext2} file system augmented by \emph{zero()} demonstrate that where their functionality overlaps, \emph{zero()} is more efficient than \emph{ftruncate()}. Additional experiments reveal that in exchange for added effective disk capacity, I/O-efficient code pays only a small performance penalty.}, } @InCollection{oldfield:bapp-pario, author = {Ron Oldfield and David Kotz}, title = {{Scientific Applications using Parallel I/O}}, booktitle = {{High Performance Mass Storage and Parallel I/O: Technologies and Applications}}, editor = {Hai Jin and Toni Cortes and Rajkumar Buyya}, year = 2001, month = {September}, chapter = 45, pages = {655--666}, publisher = {Wiley-IEEE Press}, copyright = {Wiley-IEEE Press}, ISBN13 = {978-0-471-20809-9}, URL = {https://www.cs.dartmouth.edu/~kotz/research/oldfield-bapp-pario/index.html}, abstract = {Scientific applications are increasingly being implemented on massively parallel supercomputers. Many of these applications have intense I/O demands, as well as massive computational requirements. This paper is essentially an annotated bibliography of papers and other sources of information about scientific applications using parallel I/O.}, } @Misc{kotz:pario-sw, author = {David Kotz}, title = {{Bibliography about Parallel I/O}}, howpublished = {BibTeX bibliography}, year = 2000, copyright = {David Kotz}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-pario-sw/index.html}, note = {Original version published 1994}, abstract = {A bibliography of many references on parallel I/O and multiprocessor file-systems issues. As of the fifth edition, it is available in HTML format.}, } @InCollection{kotz:encyc1, author = {David Kotz and Ravi Jain}, title = {{I/O in Parallel and Distributed Systems}}, booktitle = {{Encyclopedia of Computer Science and Technology}}, editor = {Allen Kent and James G. Williams}, year = 1999, volume = 40, chapter = 0, pages = {141--154}, publisher = {Marcel Dekker, Inc.}, copyright = {Marcel Dekker, Inc.}, ISBN13 = 9780824722937, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-encyc1/index.html}, note = {Supplement 25}, abstract = {We sketch the reasons for the I/O bottleneck in parallel and distributed systems, pointing out that it can be viewed as a special case of a general bottleneck that arises at all levels of the memory hierarchy. We argue that because of its severity, the I/O bottleneck deserves systematic attention at all levels of system design. We then present a survey of the issues raised by the I/O bottleneck in six key areas of parallel and distributed systems: applications, algorithms, languages and compilers, run-time libraries, operating systems, and architecture.}, } @Article{choudhary:sdcr, author = {Alok Choudhary and David Kotz}, title = {{Large-Scale File Systems with the Flexibility of Databases}}, journal = {ACM Computing Surveys}, year = 1996, month = {December}, volume = {28A}, number = 4, publisher = {ACM}, copyright = {ACM}, DOI = {10.1145/242224.242488}, URL = {https://www.cs.dartmouth.edu/~kotz/research/choudhary-sdcr/index.html}, note = {Position paper for the Working Group on Storage I/O for Large-Scale Computing, ACM Workshop on Strategic Directions in Computing Research. Available on-line only.}, abstract = {We note that large-scale computing includes many applications with intensive I/O demands. A data-storage system for such applications must address two issues: locating the appropriate data set, and accessing the contents of the data set. Today, there are two extreme models of data location and management: 1) file systems, which can be fast but which require a user to manage the structure of the file-name space and, often, of the file contents; and 2) object-oriented-database systems, in which even the smallest granule of data is stored as an object with associated access methods, which is very flexible but often slow. We propose a solution that may provide the performance of file systems with the flexibility of object databases.}, } @TechReport{kotz:app-pario, author = {David Kotz}, title = {{Applications of Parallel I/O}}, institution = {Dartmouth Computer Science}, year = 1996, month = {October}, number = {PCS-TR96-297}, copyright = {David Kotz}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-app-pario/index.html}, note = {Release 1}, abstract = {Scientific applications are increasingly being implemented on massively parallel supercomputers. Many of these applications have intense I/O demands, as well as massive computational requirements. This paper is essentially an annotated bibliography of papers and other sources of information about scientific applications using parallel I/O. It will be updated periodically.}, } @Misc{kotz:lecture, author = {David Kotz}, title = {{Parallel File Systems}}, howpublished = {A multimedia lecture included in the CD-ROM ``Introductory Lectures on Data-Parallel Computing'', published by AK Peters, Ltd.}, year = 1996, month = {March}, copyright = {AK Peters, Ltd.}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-lecture/index.html}, } @InCollection{kotz:pioarch, author = {David Kotz}, title = {{Introduction to Multiprocessor I/O Architecture}}, booktitle = {{Input/Output in Parallel and Distributed Computer Systems}}, editor = {Ravi Jain and John Werth and James C. Browne}, series = {The Kluwer International Series in Engineering and Computer Science}, year = 1996, volume = 362, chapter = 4, pages = {97--123}, publisher = {Kluwer Academic Publishers}, copyright = {Kluwer Academic Publishers}, ISBN13 = {978-1-4613-1401-1}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-pioarch/index.html}, abstract = {The computational performance of multiprocessors continues to improve by leaps and bounds, fueled in part by rapid improvements in processor and interconnection technology. I/O performance thus becomes ever more critical, to avoid becoming the bottleneck of system performance. In this paper we provide an introduction to I/O architectural issues in multiprocessors, with a focus on disk subsystems. While we discuss examples from actual architectures and provide pointers to interesting research in the literature, we do not attempt to provide a comprehensive survey. We concentrate on a study of the architectural design issues, and the effects of different design alternatives.}, } @TechReport{thomas:thesis, author = {Joel T. Thomas}, title = {{The Panda Array I/O Library on the Galley Parallel File System}}, institution = {Dartmouth Computer Science}, year = 1996, month = {June}, number = {PCS-TR96-288}, copyright = {the author}, address = {Hanover, NH}, URL = {https://www.cs.dartmouth.edu/~kotz/research/thomas-thesis/index.html}, note = {Available as Dartmouth Computer Science Technical Report PCS-TR96-288}, abstract = {The Panda Array I/O library, created at the University of Illinois, Urbana-Champaign, was built especially to address the needs of high-performance scientific applications. I/O has been one of the most frustrating bottlenecks to high performance for quite some time, and the Panda project is an attempt to ameliorate this problem while still providing the user with a simple, high-level interface. The Galley File System, with its hierarchical structure of files and strided requests, is another attempt at addressing the performance problem. My project was to redesign the Panda Array library for use on the Galley file system. This project involved porting Panda's three main functions: a checkpoint function for writing a large array periodically for 'safekeeping,' a restart function that would allow a checkpointed file to be read back in, and finally a timestep function that would allow the user to write a group of large arrays several times in a sequence. Panda supports several different distributions in both the compute-node memories and I/O-node disks. \par We have found that the Galley File System provides a good environment on which to build high-performance libraries, and that the mesh of Panda and Galley was a successful combination.}, } @Article{reed:panel, author = {Daniel A. Reed and Charles Catlett and Alok Choudhary and David Kotz and Marc Snir}, title = {{Parallel I/O: Getting Ready for Prime Time}}, journal = {IEEE Parallel and Distributed Technology}, year = 1995, month = {Summer}, volume = 3, number = 2, pages = {64--71}, publisher = {IEEE}, copyright = {IEEE}, DOI = {10.1109/MPDT.1995.9283668}, URL = {https://www.cs.dartmouth.edu/~kotz/research/reed-panel/index.html}, note = {Edited transcript of panel discussion at the 1994 International Conference on Parallel Processing}, abstract = {During the \emph{International Conference on Parallel Processing}, held August 15-19, 1994, we convened a panel to discuss the state of the art in parallel I/O, tools and techniques to address current problems, and challenges for the future. The following is an edited transcript of that panel.}, } @TechReport{kotz:diskmodel, author = {David Kotz and Song Bac Toh and Sriram Radhakrishnan}, title = {{A Detailed Simulation Model of the HP 97560 Disk Drive}}, institution = {Dartmouth Computer Science}, year = 1994, month = {July}, number = {PCS-TR94-220}, copyright = {the authors}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-diskmodel/index.html}, abstract = {We implemented a detailed model of the HP 97560 disk drive, to replicate a model devised by Ruemmler and Wilkes (both of Hewlett-Packard, HP). Our model simulates one or more disk drives attached to one or more SCSI buses, using a small discrete-event simulation module included in our implementation. The design is broken into three components: a test driver, the disk model itself, and the discrete-event simulation support. Thus, the disk model can be easily extracted and used in other simulation environments. We validated our model using traces obtained from HP, using the same ``demerit'' measure as Ruemmler and Wilkes. We obtained a demerit figure of 3.9\%, indicating that our model was extremely accurate. This paper describes our implementation, and is meant for those wishing to understand our model or to implement their own.}, } @Misc{kotz:diskmodel-sw, author = {David Kotz}, title = {{HP 97560 disk simulation module}}, howpublished = {Used in STARFISH and several other research projects}, year = 1994, copyright = {the author}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-diskmodel-sw/index.html}, abstract = {We implemented a detailed model of the HP 97560 disk drive, to replicate a model devised by Ruemmler and Wilkes (both of Hewlett-Packard).}, } @TechReport{cormen:integrate-tr, author = {Thomas H. Cormen and David Kotz}, title = {{Integrating Theory and Practice in Parallel File Systems}}, institution = {Dept. of Math and Computer Science, Dartmouth College}, year = 1993, month = {March}, number = {PCS-TR93-188}, copyright = {the authors}, URL = {https://www.cs.dartmouth.edu/~kotz/research/cormen-integrate-tr/index.html}, note = {Revised 9/20/94}, abstract = {Several algorithms for parallel disk systems have appeared in the literature recently, and they are asymptotically optimal in terms of the number of disk accesses. Scalable systems with parallel disks must be able to run these algorithms. We present a list of capabilities that must be provided by the system to support these optimal algorithms: control over declustering, querying about the configuration, independent I/O, turning off file caching and prefetching, and bypassing parity. We summarize recent theoretical and empirical work that justifies the need for these capabilities.}, } @InProceedings{cormen:integrate, author = {Thomas H. Cormen and David Kotz}, title = {{Integrating Theory and Practice in Parallel File Systems}}, booktitle = {{Proceedings of the Dartmouth Institute for Advanced Graduate Studies (DAGS)}}, organization = {Dartmouth Institute for Advanced Graduate Studies (DAGS)}, year = 1993, month = {June}, pages = {64--74}, publisher = {Dartmouth College}, copyright = {the authors}, address = {Hanover, NH}, URL = {https://www.cs.dartmouth.edu/~kotz/research/cormen-integrate/index.html}, note = {Revised as Dartmouth PCS-TR93-188 on 9/20/94}, abstract = {Several algorithms for parallel disk systems have appeared in the literature recently, and they are asymptotically optimal in terms of the number of disk accesses. Scalable systems with parallel disks must be able to run these algorithms. We present for the first time a list of capabilities that must be provided by the system to support these optimal algorithms: control over declustering, querying about the configuration, independent I/O, and turning off parity, file caching, and prefetching. We summarize recent theoretical and empirical work that justifies the need for these capabilities. In addition, we sketch an organization for a parallel file interface with low-level primitives and higher-level operations.}, } @InProceedings{kotz:fsint2, author = {David Kotz}, title = {{Multiprocessor File System Interfaces}}, booktitle = {{Proceedings of the International Conference on Parallel and Distributed Information Systems (PDIS)}}, year = 1993, month = {January}, pages = {194--201}, publisher = {IEEE}, copyright = {IEEE}, DOI = {10.1109/PDIS.1993.253093}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-fsint2/index.html}, abstract = {Increasingly, file systems for multiprocessors are designed with parallel access to multiple disks, to keep I/O from becoming a serious bottleneck for parallel applications. Although file system software can transparently provide high-performance access to parallel disks, a new file system interface is needed to facilitate parallel access to a file from a parallel application. We describe the difficulties faced when using the conventional (Unix-like) interface in parallel applications, and then outline ways to extend the conventional interface to provide convenient access to the file for parallel programs, while retaining the traditional interface for programs that have no need for explicitly parallel file access. Our interface includes a single naming scheme, a \emph{multiopen} operation, local and global file pointers, mapped file pointers, logical records, \emph{multifiles}, and logical coercion for backward compatibility.}, } @TechReport{kotz:throughput, author = {David Kotz}, title = {{Throughput of Existing Multiprocessor File Systems}}, institution = {Dept. of Math and Computer Science, Dartmouth College}, year = 1993, month = {May}, number = {PCS-TR93-190}, copyright = {David Kotz}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-throughput/index.html}, abstract = {Fast file systems are critical for high-performance scientific computing, since many scientific applications have tremendous I/O requirements. Many parallel supercomputers have only recently obtained fully parallel I/O architectures and file systems, which are necessary for scalable I/O performance. Scalability aside, I show here that many systems lack sufficient absolute performance. I do this by surveying the performance reported in the literature, summarized in an informal table.}, } @TechReport{kotz:fsint, author = {David Kotz}, title = {{Multiprocessor File System Interfaces}}, institution = {Dept. of Math and Computer Science, Dartmouth College}, year = 1992, month = {March}, number = {PCS-TR92-179}, copyright = {David Kotz}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-fsint/index.html}, abstract = {Increasingly, file systems for multiprocessors are designed with parallel access to multiple disks, to keep I/O from becoming a serious bottleneck for parallel applications. Although file system software can transparently provide high-performance access to parallel disks, a new file system interface is needed to facilitate parallel access to a file from a parallel application. We describe the difficulties faced when using the conventional (Unix-like) interface in parallel applications, and then outline ways to extend the conventional interface to provide convenient access to the file for parallel programs, while retaining the traditional interface for programs that have no need for explicitly parallel file access. Our interface includes a single naming scheme, a \emph{multiopen} operation, local and global file pointers, mapped file pointers, logical records, \emph{multifiles}, and logical coercion for backward compatibility.}, } @InProceedings{kotz:fsint2p, author = {David Kotz}, title = {{Multiprocessor File System Interfaces}}, booktitle = {{Proceedings of the USENIX File Systems Workshop (WOFS)}}, year = 1992, month = {May}, pages = {149--150}, publisher = {USENIX Association}, copyright = {the author}, URL = {https://www.cs.dartmouth.edu/~kotz/research/kotz-fsint2p/index.html}, } @Article{ellis:numatic, author = {C. Ellis and M. Holliday and R. LaRowe and D. Kotz and V. Khera and S. Owen and C. Connelly}, title = {{NUMAtic Project and the DUnX OS}}, journal = {IEEE Technical Committee on Operating Systems and Application Environments (Newsletter)}, year = 1991, month = {Winter}, volume = 5, number = 4, pages = {12--14}, publisher = {IEEE}, copyright = {the authors}, URL = {https://www.cs.dartmouth.edu/~kotz/research/ellis-numatic/index.html}, }