From 71b5c7028ba6be5ee394ecca200196f6f27ea84a Mon Sep 17 00:00:00 2001 From: Amanda-dong <159391549+Amanda-dong@users.noreply.github.com> Date: Tue, 23 Jun 2026 10:27:45 +0800 Subject: [PATCH 1/6] Add MDX file generation for storage finder data --- .../storage-finder-data-generator/generate.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/scripts/storage-finder-data-generator/generate.ts b/scripts/storage-finder-data-generator/generate.ts index 81f5965c6d..b07f6ae6fe 100644 --- a/scripts/storage-finder-data-generator/generate.ts +++ b/scripts/storage-finder-data-generator/generate.ts @@ -336,10 +336,41 @@ async function main(): Promise { const facetOutputPath = `${outputDirectory}/${FACET_TREE_FILENAME}`; await writeJson(serviceOutputPath, services, options.pretty); await writeJson(facetOutputPath, facetTree, options.pretty); + await writeFile( + "src/pages/storage-finder-data.mdx", + buildSearchMdx(services), + "utf8", +); logger.log(`Wrote ${services.length} services to ${serviceOutputPath}`); logger.log(`Wrote ${facetTree.length} facets to ${facetOutputPath}`); } +function stripHtml(value: string | undefined): string { + return (value ?? "") + .replaceAll(/<[^>]+>/g, " ") + .replaceAll(/\s+/g, " ") + .trim(); +} + +function buildSearchMdx(services: ServiceRecord[]): string { + return [ + "# Storage Finder Data", + "", + "This page is generated from the Storage Finder data so the search index can crawl service details.", + "", + ...services.map((service) => + [ + `## ${service.title}`, + "", + ...Object.values(service.field_data).map((field) => + [`### ${field.label}`, "", stripHtml(field.value), ""].join("\n"), + ), + ].join("\n"), + ), + "", + ].join("\n"); +} + // eslint-disable-next-line unicorn/prefer-top-level-await void (async () => { try { From 04169186eb4abf30bac31d66c374715a8e29c02e Mon Sep 17 00:00:00 2001 From: Amanda-dong <159391549+Amanda-dong@users.noreply.github.com> Date: Tue, 23 Jun 2026 10:46:30 +0800 Subject: [PATCH 2/6] Add stripHtml function and buildSearchMdx function --- .../storage-finder-data-generator/generate.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/scripts/storage-finder-data-generator/generate.ts b/scripts/storage-finder-data-generator/generate.ts index b07f6ae6fe..7e6712592a 100644 --- a/scripts/storage-finder-data-generator/generate.ts +++ b/scripts/storage-finder-data-generator/generate.ts @@ -316,6 +316,32 @@ function slugify(value: string): string { return trimmed; } +function stripHtml(value: string | undefined): string { + return (value ?? "") + .replaceAll(/<[^>]+>/g, " ") + .replaceAll(/\s+/g, " ") + .trim(); +} + +function buildSearchMdx(services: ServiceRecord[]): string { + return [ + "# Storage Finder Data", + "", + "This page is generated from the Storage Finder data so the search index can crawl service details.", + "", + ...services.map((service) => + [ + `## ${service.title}`, + "", + ...Object.values(service.field_data).map((field) => + [`### ${field.label}`, "", stripHtml(field.value), ""].join("\n"), + ), + ].join("\n"), + ), + "", + ].join("\n"); +} + async function main(): Promise { const options = parseArgs(process.argv.slice(2)); if (options.showHelp) { From 305e9c58b2efb2ebd39ccc53395809440ee8b81e Mon Sep 17 00:00:00 2001 From: Amanda-dong <159391549+Amanda-dong@users.noreply.github.com> Date: Tue, 23 Jun 2026 10:51:32 +0800 Subject: [PATCH 3/6] Remove unused stripHtml and buildSearchMdx functions Removed unused functions for HTML stripping and MDX building. --- .../storage-finder-data-generator/generate.ts | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/scripts/storage-finder-data-generator/generate.ts b/scripts/storage-finder-data-generator/generate.ts index 7e6712592a..b07f6ae6fe 100644 --- a/scripts/storage-finder-data-generator/generate.ts +++ b/scripts/storage-finder-data-generator/generate.ts @@ -316,32 +316,6 @@ function slugify(value: string): string { return trimmed; } -function stripHtml(value: string | undefined): string { - return (value ?? "") - .replaceAll(/<[^>]+>/g, " ") - .replaceAll(/\s+/g, " ") - .trim(); -} - -function buildSearchMdx(services: ServiceRecord[]): string { - return [ - "# Storage Finder Data", - "", - "This page is generated from the Storage Finder data so the search index can crawl service details.", - "", - ...services.map((service) => - [ - `## ${service.title}`, - "", - ...Object.values(service.field_data).map((field) => - [`### ${field.label}`, "", stripHtml(field.value), ""].join("\n"), - ), - ].join("\n"), - ), - "", - ].join("\n"); -} - async function main(): Promise { const options = parseArgs(process.argv.slice(2)); if (options.showHelp) { From 961b852082a7049974418e8b0c043972e80acd85 Mon Sep 17 00:00:00 2001 From: Amanda-dong <159391549+Amanda-dong@users.noreply.github.com> Date: Tue, 23 Jun 2026 11:44:49 +0800 Subject: [PATCH 4/6] Create storage-finder-data.mdx for indexing This page is intended for search indexing and is hidden from public navigation. --- src/pages/storage-finder-data.mdx | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 src/pages/storage-finder-data.mdx diff --git a/src/pages/storage-finder-data.mdx b/src/pages/storage-finder-data.mdx new file mode 100644 index 0000000000..519dd181d9 --- /dev/null +++ b/src/pages/storage-finder-data.mdx @@ -0,0 +1,9 @@ +--- +title: Storage Finder Data +hide_title: true +hide_table_of_contents: true +--- + +# Storage Finder Data + +This page is generated for search indexing and is not part of the public documentation navigation. From 87c95d6573448de4abaa13aa873923a2a601a67c Mon Sep 17 00:00:00 2001 From: Amanda-dong <159391549+Amanda-dong@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:58:33 +0800 Subject: [PATCH 5/6] Change storage finder permissions from read to write --- .github/workflows/storage-finder-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/storage-finder-sync.yml b/.github/workflows/storage-finder-sync.yml index 289153a3d4..8e37a3d025 100644 --- a/.github/workflows/storage-finder-sync.yml +++ b/.github/workflows/storage-finder-sync.yml @@ -1,7 +1,7 @@ name: Sync Storage Finder Data permissions: - contents: read + contents: write on: schedule: From 7e99f4e7cebbff264d178134140ec84e8f774aa6 Mon Sep 17 00:00:00 2001 From: Amanda-dong <159391549+Amanda-dong@users.noreply.github.com> Date: Tue, 23 Jun 2026 06:00:40 +0000 Subject: [PATCH 6/6] chore: sync storage finder data from sheet --- .../storage-finder-data-generator/generate.ts | 8 +- src/pages/storage-finder-data.mdx | 655 +++++++++++++++++- 2 files changed, 652 insertions(+), 11 deletions(-) diff --git a/scripts/storage-finder-data-generator/generate.ts b/scripts/storage-finder-data-generator/generate.ts index b07f6ae6fe..577113383c 100644 --- a/scripts/storage-finder-data-generator/generate.ts +++ b/scripts/storage-finder-data-generator/generate.ts @@ -337,10 +337,10 @@ async function main(): Promise { await writeJson(serviceOutputPath, services, options.pretty); await writeJson(facetOutputPath, facetTree, options.pretty); await writeFile( - "src/pages/storage-finder-data.mdx", - buildSearchMdx(services), - "utf8", -); + "src/pages/storage-finder-data.mdx", + buildSearchMdx(services), + "utf8", + ); logger.log(`Wrote ${services.length} services to ${serviceOutputPath}`); logger.log(`Wrote ${facetTree.length} facets to ${facetOutputPath}`); } diff --git a/src/pages/storage-finder-data.mdx b/src/pages/storage-finder-data.mdx index 519dd181d9..356a09053c 100644 --- a/src/pages/storage-finder-data.mdx +++ b/src/pages/storage-finder-data.mdx @@ -1,9 +1,650 @@ ---- -title: Storage Finder Data -hide_title: true -hide_table_of_contents: true ---- - # Storage Finder Data -This page is generated for search indexing and is not part of the public documentation navigation. +This page is generated from the Storage Finder data so the search index can crawl service details. + +## Amazon S3 + +### Links + +Amazon S3 + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +Archive and long-term, high-security storage of large datasets. + +### Limitations + +AWS S3 is a chargeback service and an NYU chartfield is required for chargeback. Max individual file upload size is 5TB. + +### Permission Settings + +Restricted access by individual or group. + +### Eligibility + +Faculty; Staff + +### Access Locations + +All + +### Backup + +Yes, available for additional cost. + +### Additional Features + +access controls, automated workflows, data replication, instrument data transfer, large data transfers, version control + +## HSRN Ceph + +### Links + +HSRN Ceph + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Long term storage + +### Limitations + +Not Available + +### Permission Settings + +Restricted access by individual or group using facl (Linux access control lists) and Grouper for access and group membership. + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +Yes, 30 day retention + +### Additional Features + +access controls + +## File Sharing (Windows) + +### Links + +File Sharing (Windows) + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +NYU departments that want to store & share files of low & moderate security + +### Limitations + +Workstations will have to be joined to NYU's Active Directory + +### Permission Settings + +Options for read only, modify, or remove. Activated in NYU's Active Directory security groups via role-based access. + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +VPN + +### Backup + +Yes, backed up daily, retains last 30 days; monitored 24x7 + +### Additional Features + +access controls, automated workflows, instrument data transfer, large data transfers, version control + +## HPC Archive + +### Links + +HPC Archive + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Long term storage + +### Limitations + +2TB limit + +### Permission Settings + +NetID group permissions using facl + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +Yes, 30 day retention + +### Additional Features + +access controls + +## HPC Research Project Space + +### Links + +HPC Research Project Space + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Use of the high-security HPC environment to actively analyze research data. + +### Limitations + +Storage space and number of inodes (files) are available for a fee. No limit on the amount of storage that can be purchased, but significant purchases may require approval. + +### Permission Settings + +Restricted access by individual or group using Access Control Lists (ACLs). Sharing with external collaborators is possible using Globus Data Transfer Node . + +### Eligibility + +Faculty; and Graduate Student HPC user account holders + +### Access Locations + +VPN + +### Backup + +Yes, retains past 30 days. + +### Additional Features + +access controls, large data transfers + +## HPC Scratch + +### Links + +HPC Scratch + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Temporary storage + +### Limitations + +2TB limit with 60 day purge policy for untouched files + +### Permission Settings + +NetID group permissions using facl + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +No + +### Additional Features + +access controls + +## NYU Box + +### Links + +NYU Box + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +Storage of NYU data with high security & the ability to collaborate with external associates + +### Limitations + +Max upload is 50 GB for an single file + +### Permission Settings + +NetID group permissions granted by NYU IT Admin + +### Eligibility + +Faculty; Staff; Faculty sponsored; External collaborators + +### Access Locations + +All + +### Backup + +Yes, Retains up to 100 previous versions of a single file + +### Additional Features + +access controls + +## NYU Google Drive + +### Links + +NYU Google Drive + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Storage of NYU data with moderate security & the ability to collaborate with external associates + +### Limitations + +My Drive: max storage is 20 GB Shared Drives: max storage TBD + +### Permission Settings + +Shared link access for view, comment, edit and temp access to comment and view + +### Eligibility + +Faculty; Staff; Students; External collaborators + +### Access Locations + +All + +### Backup + +Yes, retains last 30 days or 100 revisions + +### Additional Features + +access controls + +## NYU Shared Google Drive + +### Links + +NYU Shared Google Drive + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Storage of NYU data with moderate security & the ability to collaborate with external associates + +### Limitations + +My Drive: max storage is 20 GB Shared Drives: max storage TBD + +### Permission Settings + +Shared link access for view, comment, edit and temp access to comment and view + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +All + +### Backup + +Yes, retains last 30 days or 100 revisions + +### Additional Features + +access controls, version control + +## NYU Stream + +### Links + +NYU Stream + +### Storable Files + +Low Risk + +### Use Case + +Video and audio + +### Limitations + +User must have nyu.edu account; Student must be actively enrolled + +### Permission Settings + +Options to restrict access to content by individual or group + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +VPN + +### Backup + +Yes + +### Additional Features + +access controls + +## Qualtrics + +### Links + +Qualtrics + +### Storable Files + +Moderate & Low Risk + +### Use Case + +To create and administer web surveys for research, teaching and administrative purposes + +### Limitations + +No limit for storage, 10 GB export limit per CSV file + +### Permission Settings + +One owner, adjustable group collaboration permissions for teamwork + +### Eligibility + +Faculty; Retired Faculty; Staff; Students; External collaborators + +### Access Locations + +VPN, Browser GUI + +### Backup + +Yes, Yes, backed up daily by Qualtrics + +### Additional Features + +access controls, automated workflows, version control + +## REDCap + +### Links + +REDCap + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +A secure web application for managing online surveys that involves collection of High Risk data and in compliance with multiple government standards + +### Limitations + +Small to medium sized projects + +### Permission Settings + +Customizable rights, roles and access to data can be given for netid collaborators with an approved NYU REDCap account + +### Eligibility + +Faculty; Staff; Faculty sponsored; External collaborators + +### Access Locations + +VPN, Browser GUI + +### Backup + +Yes, backed up daily. + +### Additional Features + +access controls + +## Research Data Lake (Pilot) + +### Links + +Research Data Lake (Pilot) + +### Storable Files + +Moderate & Low Risk + +### Use Case + +High-performance storage for active research datasets with multiple users. Supports structured and semi-structured data + +### Limitations + +During pilot phase, projects must be approved by RTS staff + +### Permission Settings + +Project dependent, role-based permissions. NeiID users added by PI request + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +Yes, available as configurable snapshotting + +### Additional Features + +access controls + +## Research Workspace + +### Links + +Research Workspace + +### Storable Files + +Moderate & Low Risk + +### Use Case + +Mountable storage with a focus on fast access to actively used datasets. + +### Limitations + +VPN needed when off-campus. + +### Permission Settings + +Done via Grouper; can give collaborators read access, read/write access, or administrator priveleges for a share. + +### Eligibility + +Faculty; Staff; Faculty sponsored + +### Access Locations + +VPN + +### Backup + +Yes, Retains last 30 days + +### Additional Features + +access controls, large data transfer + +## Secure Research Data Environment + +### Links + +Secure Research Data Environment + +### Storable Files + +High, Moderate & Low Risk + +### Use Case + +Researchers using the HPC clusters to data analysis of simulations sharing data amongst group members + +### Limitations + +Requires secure data stewardship knowledge + +### Permission Settings + +Project dependent, role-based permissions, requires data steward role and netid + +### Eligibility + +Faculty; Faculty sponsored + +### Access Locations + +VPN, Public Cloud, Browser GUI + +### Backup + +No + +### Additional Features + +access controls, automated workflows, code sharing, large data transfers, version control + +## Ultraviolet Repository + +### Links + +Ultraviolet Repository + +### Storable Files + +Moderate & Low Risk + +### Use Case + +A repository for scholarly materials which are deposited for open access publication and long-term preservation + +### Limitations + +Recommended to 100 files or 50GB (per file), contact repository team uv@nyu.edu if exceeds this limit + +### Permission Settings + +Public visibility by default + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +All + +### Backup + +Yes, backed up as different versions + +### Additional Features + +code sharing, data replication, DOI generation, version control + +## Faculty Digital Archive + +### Links + +Faculty Digital Archive + +### Storable Files + +Moderate & Low Risk + +### Use Case + +A repository for scholarly materials which are deposited for open access publication and long-term preservation + +### Limitations + +Not Available + +### Permission Settings + +Public visibility by default + +### Eligibility + +Faculty; Staff; Students + +### Access Locations + +All + +### Backup + +Yes + +### Additional Features + +code sharing, data replication +