Skip to content

Repo sync #38479

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 22, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
"start": "cross-env NODE_ENV=development ENABLED_LANGUAGES=en nodemon src/frame/server.ts",
"start-all-languages": "cross-env NODE_ENV=development tsx src/frame/server.ts",
"start-for-playwright": "cross-env ROOT=src/fixtures/fixtures TRANSLATIONS_FIXTURE_ROOT=src/fixtures/fixtures/translations ENABLED_LANGUAGES=en,ja NODE_ENV=test tsx src/frame/server.ts",
"symlink-from-local-repo": "tsx src/early-access/scripts/symlink-from-local-repo.js",
"symlink-from-local-repo": "tsx src/early-access/scripts/symlink-from-local-repo.ts",
"sync-audit-log": "tsx src/audit-logs/scripts/sync.ts",
"sync-codeql-cli": "tsx src/codeql-cli/scripts/sync.js",
"sync-graphql": "tsx src/graphql/scripts/sync.js",
Expand All @@ -98,13 +98,13 @@
"test-moved-content": "tsx src/content-render/scripts/test-moved-content.ts",
"tsc": "tsc --noEmit",
"unallowed-contributions": "tsx src/workflows/unallowed-contributions.ts",
"update-data-and-image-paths": "tsx src/early-access/scripts/update-data-and-image-paths.js",
"update-data-and-image-paths": "tsx src/early-access/scripts/update-data-and-image-paths.ts",
"update-enterprise-dates": "tsx src/ghes-releases/scripts/update-enterprise-dates.ts",
"update-internal-links": "tsx src/links/scripts/update-internal-links.ts",
"validate-asset-images": "tsx src/assets/scripts/validate-asset-images.ts",
"validate-github-github-docs-urls": "tsx src/links/scripts/validate-github-github-docs-urls/index.ts",
"warmup-remotejson": "tsx src/archives/scripts/warmup-remotejson.ts",
"what-docs-early-access-branch": "tsx src/early-access/scripts/what-docs-early-access-branch.js"
"what-docs-early-access-branch": "tsx src/early-access/scripts/what-docs-early-access-branch.ts"
},
"lint-staged": {
"*.{js,mjs,ts,tsx}": "eslint --cache --fix",
Expand Down
146 changes: 84 additions & 62 deletions ...s/scripts/migrate-early-access-product.js → ...s/scripts/migrate-early-access-product.ts
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ import { program } from 'commander'
import { execFileSync } from 'child_process'
import frontmatter from '#src/frame/lib/read-frontmatter.js'
import patterns from '#src/frame/lib/patterns.js'
import addRedirectToFrontmatter from '#src/redirects/scripts/helpers/add-redirect-to-frontmatter.js'
import addRedirectToFrontmatter from '@/redirects/scripts/helpers/add-redirect-to-frontmatter'
import walkFiles from '#src/workflows/walk-files.ts'

const contentFiles = walkFiles('content', '.md', { includeEarlyAccess: true })
const contentDir = path.posix.join(process.cwd(), 'content')
const contentFiles: string[] = walkFiles('content', '.md', { includeEarlyAccess: true })
const contentDir: string = path.posix.join(process.cwd(), 'content')

program
.description('Move a product-level early access docs set to a category level.')
Expand All @@ -34,29 +34,30 @@ program
)
.parse(process.argv)

const oldPathId = program.opts().oldPath.replace('content/', '')
const newPathId = program.opts().newPath.replace('content/', '')
const oldPathId: string = program.opts().oldPath.replace('content/', '')
const newPathId: string = program.opts().newPath.replace('content/', '')

const oldPath = path.posix.join(contentDir, oldPathId)
const newPath = path.posix.join(contentDir, newPathId)
const oldPath: string = path.posix.join(contentDir, oldPathId)
const newPath: string = path.posix.join(contentDir, newPathId)

if (!fs.existsSync(oldPath)) {
console.error(`Error! Can't find ${oldPath}`)
process.exit(1)
}

const filesToMigrate = contentFiles.filter((file) => file.includes(`/${oldPathId}/`))
const filesToMigrate: string[] = contentFiles.filter((file) => file.includes(`/${oldPathId}/`))

if (!filesToMigrate.length) {
console.error(`Error! Can't find any files in ${oldPath}`)
process.exit(1)
}

const migratePath = path.posix.join(contentDir, newPathId)
const migratePath: string = path.posix.join(contentDir, newPathId)

// 1. Update the image and data refs in the to-be-migrated early access files BEFORE moving them.
try {
execFileSync('src/early-access/scripts/update-data-and-image-paths.js', [
execFileSync('tsx', [
'src/early-access/scripts/update-data-and-image-paths.ts',
'-p',
`content/${oldPathId}`,
'--remove',
Expand All @@ -66,28 +67,30 @@ try {
process.exit(1)
}

const variablesToMove = []
const reusablesToMove = []
const imagesToMove = []
const variablesToMove: string[] = []
const reusablesToMove: string[] = []
const imagesToMove: string[] = []

// 2. Add redirects to and update frontmatter in the to-be-migrated early access files BEFORE moving them.
filesToMigrate.forEach((filepath) => {
const { content, data } = frontmatter(fs.readFileSync(filepath, 'utf8'))
const redirectString = filepath
const redirectString: string = filepath
.replace('content/', '/')
.replace('/index.md', '')
.replace('.md', '')
data.redirect_from = addRedirectToFrontmatter(data.redirect_from, redirectString)
delete data.hidden
delete data.noEarlyAccessBanner
delete data.earlyAccessToc
fs.writeFileSync(filepath, frontmatter.stringify(content, data, { lineWidth: 10000 }))
if (data) {
data.redirect_from = addRedirectToFrontmatter(data.redirect_from, redirectString)
delete data.hidden
delete data.noEarlyAccessBanner
delete data.earlyAccessToc
fs.writeFileSync(filepath, frontmatter.stringify(content || '', data))
}

// 4. Find the data files and images referenced in the early access files so we can move them over.
const dataRefs = content.match(patterns.dataReference) || []
const variables = dataRefs.filter((ref) => ref.includes('variables'))
const reusables = dataRefs.filter((ref) => ref.includes('reusables'))
const images = content.match(patterns.imagePath) || []
const dataRefs: string[] = content ? content.match(patterns.dataReference) || [] : []
const variables: string[] = dataRefs.filter((ref) => ref.includes('variables'))
const reusables: string[] = dataRefs.filter((ref) => ref.includes('reusables'))
const images: string[] = content ? content.match(patterns.imagePath) || [] : []

variablesToMove.push(...variables)
reusablesToMove.push(...reusables)
Expand All @@ -103,73 +106,78 @@ Array.from(new Set(imagesToMove)).forEach((imageRef) => moveImage(imageRef))
execFileSync('mv', [oldPath, migratePath])

// 5. Update the parent product TOC with the new child path.
const parentProductTocPath = path.posix.join(path.dirname(newPath), 'index.md')
const parentProducToc = frontmatter(fs.readFileSync(parentProductTocPath, 'utf-8'))
parentProducToc.data.children.push(`/${path.basename(newPathId)}`)
const parentProductTocPath: string = path.posix.join(path.dirname(newPath), 'index.md')
const parentProductToc = frontmatter(fs.readFileSync(parentProductTocPath, 'utf-8'))
if (parentProductToc.data && Array.isArray(parentProductToc.data.children)) {
parentProductToc.data.children.push(`/${path.basename(newPathId)}`)
}

fs.writeFileSync(
parentProductTocPath,
frontmatter.stringify(parentProducToc.content, parentProducToc.data, { lineWidth: 10000 }),
frontmatter.stringify(parentProductToc.content || '', parentProductToc.data || {}),
)

// 6. Optionally, update the new product TOC with the new title.
if (program.opts().newTitle) {
const productTocPath = path.posix.join(newPath, 'index.md')
const productTocPath: string = path.posix.join(newPath, 'index.md')
const productToc = frontmatter(fs.readFileSync(productTocPath, 'utf-8'))
productToc.data.title = program.opts().newTitle
if (productToc.data) {
productToc.data.title = program.opts().newTitle
}

fs.writeFileSync(
productTocPath,
frontmatter.stringify(productToc.content, productToc.data, { lineWidth: 10000 }),
frontmatter.stringify(productToc.content || '', productToc.data || {}),
)
}

// 7. Update internal links now that the files have been moved.
console.log('\nRunning script to update internal links...')
execFileSync('src/links/scripts/update-internal-links.js')
execFileSync('tsx', ['src/links/scripts/update-internal-links.ts'])

console.log(`
Done! Did the following:
- Moved content/${oldPathId} files to content/${newPathId}
- Ran ./src/early-access/scripts/update-data-and-images-paths.js
- Ran ./src/early-access/scripts/update-data-and-image-paths.ts
- Added redirects to the moved files
- Updated children frontmatter entries in index.md files
- Ran ./src/links/scripts/update-internal-links.js
- Ran ./src/links/scripts/update-internal-links.ts

Please review all the changes in docs-internal and docs-early-access, especially to index.md files. You may need to do some manual cleanup.
`)

function moveVariable(dataRef) {
function moveVariable(dataRef: string): void {
// Get the data filepath from the data reference,
// where the data reference looks like: {% data variables.foo.bar %}
// and the data filepath looks like: data/variables/foo.yml with key of 'bar'.
const variablePathArray = dataRef
.match(/{% (?:data|indented_data_reference) (.*?) %}/)[1]
.split('.')
// If early access is part of the path, remove it (since the path below already includes it)
.filter((n) => n !== 'early-access')
const variablePathArray: string[] =
dataRef
.match(/{% (?:data|indented_data_reference) (.*?) %}/)?.[1]
.split('.')
// If early access is part of the path, remove it (since the path below already includes it)
.filter((n) => n !== 'early-access') || []

// Given a string `variables.foo.bar` split into an array, we want the last segment 'bar', which is the variable key.
// Then pop 'bar' off the array because it's not really part of the filepath.
// The filepath we want is `variables/foo.yml`.
const variableKey = last(variablePathArray)
const variableKey: string = last(variablePathArray) as string

variablePathArray.pop()

const oldVariablePath = path.posix.join(
const oldVariablePath: string = path.posix.join(
process.cwd(),
'data/early-access',
`${variablePathArray.join('/')}.yml`,
)
const newVariablePath = path.posix.join(
const newVariablePath: string = path.posix.join(
process.cwd(),
'data',
`${variablePathArray.join('/')}.yml`,
)
const nonAltPath = newVariablePath.replace('-alt.yml', '.yml')
const oldAltPath = oldVariablePath.replace('.yml', '-alt.yml')
const nonAltPath: string = newVariablePath.replace('-alt.yml', '.yml')
const oldAltPath: string = oldVariablePath.replace('.yml', '-alt.yml')

let oldPath = oldVariablePath
let oldPath: string = oldVariablePath

// If the old variable path doesn't exist, assume no migration needed.
if (!fs.existsSync(oldVariablePath)) {
Expand All @@ -184,12 +192,17 @@ function moveVariable(dataRef) {
}
}

const variableFileContent = yaml.load(fs.readFileSync(oldPath, 'utf8'))
const value = variableFileContent[variableKey]
const variableFileContent: Record<string, any> = yaml.load(
fs.readFileSync(oldPath, 'utf8'),
) as Record<string, any>
const value: any = variableFileContent[variableKey]

// If the variable file already exists, add the key/value pair.
if (fs.existsSync(nonAltPath)) {
const content = yaml.load(fs.readFileSync(nonAltPath, 'utf8'))
const content: Record<string, any> = yaml.load(fs.readFileSync(nonAltPath, 'utf8')) as Record<
string,
any
>
if (!content[variableKey]) {
const newString = `\n\n${variableKey}: ${value}`
fs.appendFileSync(nonAltPath, newString)
Expand All @@ -199,19 +212,24 @@ function moveVariable(dataRef) {
}
}

function moveReusable(dataRef) {
function moveReusable(dataRef: string): void {
// Get the data filepath from the data reference,
// where the data reference looks like: {% data reusables.foo.bar %}
// and the data filepath looks like: data/reusables/foo/bar.md.
const reusablePath = dataRef
.match(/{% (?:data|indented_data_reference) (\S*?) .*%}/)[1]
.split('.')
// If early access is part of the path, remove it (since the path below already includes it)
.filter((n) => n !== 'early-access')
.join('/')

const oldReusablePath = path.posix.join(process.cwd(), 'data/early-access', `${reusablePath}.md`)
const newReusablePath = path.posix.join(process.cwd(), 'data', `${reusablePath}.md`)
const reusablePath: string =
dataRef
.match(/{% (?:data|indented_data_reference) (\S*?) .*%}/)?.[1]
.split('.')
// If early access is part of the path, remove it (since the path below already includes it)
.filter((n) => n !== 'early-access')
.join('/') || ''

const oldReusablePath: string = path.posix.join(
process.cwd(),
'data/early-access',
`${reusablePath}.md`,
)
const newReusablePath: string = path.posix.join(process.cwd(), 'data', `${reusablePath}.md`)

// If the old reusable path doesn't exist, assume no migration needed.
if (!fs.existsSync(oldReusablePath)) {
Expand All @@ -229,14 +247,18 @@ function moveReusable(dataRef) {
}
}

function moveImage(imageRef) {
const imagePath = imageRef
function moveImage(imageRef: string): void {
const imagePath: string = imageRef
.replace('/assets/images/', '')
// If early access is part of the path, remove it (since the path below already includes it)
.replace('early-access', '')

const oldImagePath = path.posix.join(process.cwd(), 'assets/images/early-access', imagePath)
const newImagePath = path.posix.join(process.cwd(), 'assets/images', imagePath)
const oldImagePath: string = path.posix.join(
process.cwd(),
'assets/images/early-access',
imagePath,
)
const newImagePath: string = path.posix.join(process.cwd(), 'assets/images', imagePath)

// If the old image path doesn't exist, assume no migration needed.
if (!fs.existsSync(oldImagePath)) {
Expand Down
34 changes: 21 additions & 13 deletions ...access/scripts/symlink-from-local-repo.js → ...access/scripts/symlink-from-local-repo.ts
100755 → 100644
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ const earlyAccessRepo = 'docs-early-access'
const earlyAccessDirName = 'early-access'
const earlyAccessRepoUrl = `https://github.com/github/${earlyAccessRepo}`

interface ProgramOptions {
pathToEarlyAccessRepo?: string
unlink?: boolean
}

program
.description(`Create or destroy symlinks to your local "${earlyAccessRepo}" repository.`)
.option(
Expand All @@ -25,44 +30,45 @@ program
.option('-u, --unlink', 'remove the symlinks')
.parse(process.argv)

const { pathToEarlyAccessRepo, unlink } = program.opts()
const { pathToEarlyAccessRepo, unlink }: ProgramOptions = program.opts()

if (!pathToEarlyAccessRepo && !unlink) {
throw new Error('Must provide either `--path-to-early-access-repo <PATH>` or `--unlink`')
}

let earlyAccessLocalRepoDir
let earlyAccessLocalRepoDir: string | undefined

// If creating symlinks, run some extra validation
if (!unlink && pathToEarlyAccessRepo) {
earlyAccessLocalRepoDir = path.resolve(process.cwd(), pathToEarlyAccessRepo)

let dirStats
let dirStats: fs.Stats | null
try {
dirStats = fs.statSync(earlyAccessLocalRepoDir)
} catch (err) {
} catch {
dirStats = null
}

if (!dirStats) {
throw new Error(
`The local "${earlyAccessRepo}" repo directory does not exist:`,
earlyAccessLocalRepoDir,
`The local "${earlyAccessRepo}" repo directory does not exist: ${earlyAccessLocalRepoDir}`,
)
}
if (dirStats && !dirStats.isDirectory()) {
throw new Error(
`A non-directory entry exists at the local "${earlyAccessRepo}" repo directory location:`,
earlyAccessLocalRepoDir,
`A non-directory entry exists at the local "${earlyAccessRepo}" repo directory location: ${earlyAccessLocalRepoDir}`,
)
}
}

const destinationDirNames = ['content', 'data', 'assets/images']
const destinationDirsMap = destinationDirNames.reduce((map, dirName) => {
map[dirName] = path.join(process.cwd(), dirName, earlyAccessDirName)
return map
}, {})
const destinationDirNames: string[] = ['content', 'data', 'assets/images']
const destinationDirsMap: Record<string, string> = destinationDirNames.reduce(
(map, dirName) => {
map[dirName] = path.join(process.cwd(), dirName, earlyAccessDirName)
return map
},
{} as Record<string, string>,
)

// Remove all existing early access directories from this repo
destinationDirNames.forEach((dirName) => {
Expand All @@ -82,6 +88,8 @@ if (unlink) {

// Move the latest early access source directories into this repo
destinationDirNames.forEach((dirName) => {
if (!earlyAccessLocalRepoDir) return

const sourceDir = path.join(earlyAccessLocalRepoDir, dirName)
const destDir = destinationDirsMap[dirName]

Expand Down
Loading
Loading