diff --git a/.github/workflows/storage-finder-sync.yml b/.github/workflows/storage-finder-sync.yml index 289153a3d4..8e37a3d025 100644 --- a/.github/workflows/storage-finder-sync.yml +++ b/.github/workflows/storage-finder-sync.yml @@ -1,7 +1,7 @@ name: Sync Storage Finder Data permissions: - contents: read + contents: write on: schedule: diff --git a/scripts/storage-finder-data-generator/generate.ts b/scripts/storage-finder-data-generator/generate.ts index 81f5965c6d..577113383c 100644 --- a/scripts/storage-finder-data-generator/generate.ts +++ b/scripts/storage-finder-data-generator/generate.ts @@ -336,10 +336,41 @@ async function main(): Promise { const facetOutputPath = `${outputDirectory}/${FACET_TREE_FILENAME}`; await writeJson(serviceOutputPath, services, options.pretty); await writeJson(facetOutputPath, facetTree, options.pretty); + await writeFile( + "src/pages/storage-finder-data.mdx", + buildSearchMdx(services), + "utf8", + ); logger.log(`Wrote ${services.length} services to ${serviceOutputPath}`); logger.log(`Wrote ${facetTree.length} facets to ${facetOutputPath}`); } +function stripHtml(value: string | undefined): string { + return (value ?? "") + .replaceAll(/<[^>]+>/g, " ") + .replaceAll(/\s+/g, " ") + .trim(); +} + +function buildSearchMdx(services: ServiceRecord[]): string { + return [ + "# Storage Finder Data", + "", + "This page is generated from the Storage Finder data so the search index can crawl service details.", + "", + ...services.map((service) => + [ + `## ${service.title}`, + "", + ...Object.values(service.field_data).map((field) => + [`### ${field.label}`, "", stripHtml(field.value), ""].join("\n"), + ), + ].join("\n"), + ), + "", + ].join("\n"); +} + // eslint-disable-next-line unicorn/prefer-top-level-await void (async () => { try { diff --git a/src/pages/storage-finder-data.mdx b/src/pages/storage-finder-data.mdx new file mode 100644 index 0000000000..356a09053c --- /dev/null +++ b/src/pages/storage-finder-data.mdx @@ -0,0 +1,650 @@ +# Storage Finder Data + +This page is generated from the Storage Finder data so the search index can crawl service details. + +## Amazon S3 + +### Links + +Amazon S3 + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +Archive and long-term, high-security storage of large datasets. + +### Limitations + +AWS S3 is a chargeback service and an NYU chartfield is required for chargeback. Max individual file upload size is 5TB. + +### Permission Settings + +Restricted access by individual or group. + +### Eligibility + +Faculty; Staff + +### Access Locations + +All + +### Backup + +Yes, available for additional cost. + +### Additional Features + +access controls, automated workflows, data replication, instrument data transfer, large data transfers, version control + +## HSRN Ceph + +### Links + +HSRN Ceph + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Long term storage + +### Limitations + +Not Available + +### Permission Settings + +Restricted access by individual or group using facl (Linux access control lists) and Grouper for access and group membership. + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +Yes, 30 day retention + +### Additional Features + +access controls + +## File Sharing (Windows) + +### Links + +File Sharing (Windows) + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +NYU departments that want to store & share files of low & moderate security + +### Limitations + +Workstations will have to be joined to NYU's Active Directory + +### Permission Settings + +Options for read only, modify, or remove. Activated in NYU's Active Directory security groups via role-based access. + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +VPN + +### Backup + +Yes, backed up daily, retains last 30 days; monitored 24x7 + +### Additional Features + +access controls, automated workflows, instrument data transfer, large data transfers, version control + +## HPC Archive + +### Links + +HPC Archive + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Long term storage + +### Limitations + +2TB limit + +### Permission Settings + +NetID group permissions using facl + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +Yes, 30 day retention + +### Additional Features + +access controls + +## HPC Research Project Space + +### Links + +HPC Research Project Space + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Use of the high-security HPC environment to actively analyze research data. + +### Limitations + +Storage space and number of inodes (files) are available for a fee. No limit on the amount of storage that can be purchased, but significant purchases may require approval. + +### Permission Settings + +Restricted access by individual or group using Access Control Lists (ACLs). Sharing with external collaborators is possible using Globus Data Transfer Node . + +### Eligibility + +Faculty; and Graduate Student HPC user account holders + +### Access Locations + +VPN + +### Backup + +Yes, retains past 30 days. + +### Additional Features + +access controls, large data transfers + +## HPC Scratch + +### Links + +HPC Scratch + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Temporary storage + +### Limitations + +2TB limit with 60 day purge policy for untouched files + +### Permission Settings + +NetID group permissions using facl + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +No + +### Additional Features + +access controls + +## NYU Box + +### Links + +NYU Box + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +Storage of NYU data with high security & the ability to collaborate with external associates + +### Limitations + +Max upload is 50 GB for an single file + +### Permission Settings + +NetID group permissions granted by NYU IT Admin + +### Eligibility + +Faculty; Staff; Faculty sponsored; External collaborators + +### Access Locations + +All + +### Backup + +Yes, Retains up to 100 previous versions of a single file + +### Additional Features + +access controls + +## NYU Google Drive + +### Links + +NYU Google Drive + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Storage of NYU data with moderate security & the ability to collaborate with external associates + +### Limitations + +My Drive: max storage is 20 GB Shared Drives: max storage TBD + +### Permission Settings + +Shared link access for view, comment, edit and temp access to comment and view + +### Eligibility + +Faculty; Staff; Students; External collaborators + +### Access Locations + +All + +### Backup + +Yes, retains last 30 days or 100 revisions + +### Additional Features + +access controls + +## NYU Shared Google Drive + +### Links + +NYU Shared Google Drive + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Storage of NYU data with moderate security & the ability to collaborate with external associates + +### Limitations + +My Drive: max storage is 20 GB Shared Drives: max storage TBD + +### Permission Settings + +Shared link access for view, comment, edit and temp access to comment and view + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +All + +### Backup + +Yes, retains last 30 days or 100 revisions + +### Additional Features + +access controls, version control + +## NYU Stream + +### Links + +NYU Stream + +### Storable Files + +Low Risk + +### Use Case + +Video and audio + +### Limitations + +User must have nyu.edu account; Student must be actively enrolled + +### Permission Settings + +Options to restrict access to content by individual or group + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +VPN + +### Backup + +Yes + +### Additional Features + +access controls + +## Qualtrics + +### Links + +Qualtrics + +### Storable Files + +Moderate & Low Risk + +### Use Case + +To create and administer web surveys for research, teaching and administrative purposes + +### Limitations + +No limit for storage, 10 GB export limit per CSV file + +### Permission Settings + +One owner, adjustable group collaboration permissions for teamwork + +### Eligibility + +Faculty; Retired Faculty; Staff; Students; External collaborators + +### Access Locations + +VPN, Browser GUI + +### Backup + +Yes, Yes, backed up daily by Qualtrics + +### Additional Features + +access controls, automated workflows, version control + +## REDCap + +### Links + +REDCap + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +A secure web application for managing online surveys that involves collection of High Risk data and in compliance with multiple government standards + +### Limitations + +Small to medium sized projects + +### Permission Settings + +Customizable rights, roles and access to data can be given for netid collaborators with an approved NYU REDCap account + +### Eligibility + +Faculty; Staff; Faculty sponsored; External collaborators + +### Access Locations + +VPN, Browser GUI + +### Backup + +Yes, backed up daily. + +### Additional Features + +access controls + +## Research Data Lake (Pilot) + +### Links + +Research Data Lake (Pilot) + +### Storable Files + +Moderate & Low Risk + +### Use Case + +High-performance storage for active research datasets with multiple users. Supports structured and semi-structured data + +### Limitations + +During pilot phase, projects must be approved by RTS staff + +### Permission Settings + +Project dependent, role-based permissions. NeiID users added by PI request + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +Yes, available as configurable snapshotting + +### Additional Features + +access controls + +## Research Workspace + +### Links + +Research Workspace + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Mountable storage with a focus on fast access to actively used datasets. + +### Limitations + +VPN needed when off-campus. + +### Permission Settings + +Done via Grouper; can give collaborators read access, read/write access, or administrator priveleges for a share. + +### Eligibility + +Faculty; Staff; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +Yes, Retains last 30 days + +### Additional Features + +access controls, large data transfer + +## Secure Research Data Environment + +### Links + +Secure Research Data Environment + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +Researchers using the HPC clusters to data analysis of simulations sharing data amongst group members + +### Limitations + +Requires secure data stewardship knowledge + +### Permission Settings + +Project dependent, role-based permissions, requires data steward role and netid + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN, Public Cloud, Browser GUI + +### Backup + +No + +### Additional Features + +access controls, automated workflows, code sharing, large data transfers, version control + +## Ultraviolet Repository + +### Links + +Ultraviolet Repository + +### Storable Files + +Moderate & Low Risk + +### Use Case + +A repository for scholarly materials which are deposited for open access publication and long-term preservation + +### Limitations + +Recommended to 100 files or 50GB (per file), contact repository team uv@nyu.edu if exceeds this limit + +### Permission Settings + +Public visibility by default + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +All + +### Backup + +Yes, backed up as different versions + +### Additional Features + +code sharing, data replication, DOI generation, version control + +## Faculty Digital Archive + +### Links + +Faculty Digital Archive + +### Storable Files + +Moderate & Low Risk + +### Use Case + +A repository for scholarly materials which are deposited for open access publication and long-term preservation + +### Limitations + +Not Available + +### Permission Settings + +Public visibility by default + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +All + +### Backup + +Yes + +### Additional Features + +code sharing, data replication +