Better domain verification

- deletes certificates, if any, when a domain is in HOLD
- better initial domain structure
- also deletes related jobs on domain deletion
- fix ACM validation values gathering
- put domain in HOLD after 3 consecutive critical errors
- independent verification steps
- fix typo on domain update via domain verification

Author: Soxasora

api/acm/index.js

@@ -41,3 +41,13 @@ export async function getCertificateStatus (certificateArn) {
   const certificate = await describeCertificate(certificateArn)
   return certificate.Certificate.Status
 }
+
+export async function deleteCertificate (certificateArn) {
+  if (process.env.NODE_ENV === 'development') {
+    config.endpoint = process.env.LOCALSTACK_ENDPOINT
+  }
+  const acm = new AWS.ACM(config)
+  const result = await acm.deleteCertificate({ CertificateArn: certificateArn }).promise()
+  console.log(`delete certificate attempt for ${certificateArn}, result: ${JSON.stringify(result)}`)
+  return result
+}

api/resolvers/domain.js

@@ -33,21 +33,25 @@ export default {
         throw new GqlInputError('invalid domain format')
       }
 
+      const existing = await models.customDomain.findUnique({ where: { subName } })
+
       if (domain) {
-        const existing = await models.customDomain.findUnique({ where: { subName } })
         if (existing && existing.domain === domain && existing.status !== 'HOLD') {
           throw new GqlInputError('domain already set')
         }
 
         const initializeDomain = {
           domain,
+          createdAt: new Date(),
           status: 'PENDING',
           verification: {
             dns: {
               state: 'PENDING',
               cname: 'stacker.news',
               // generate a random txt record only if it's a new domain
-              txt: existing?.domain === domain ? existing.verification.dns.txt : randomBytes(32).toString('base64')
+              txt: existing?.domain === domain && existing.verification.dns.txt
+                ? existing.verification.dns.txt
+                : randomBytes(32).toString('base64')
             },
             ssl: {
               state: 'WAITING',
@@ -72,15 +76,24 @@ export default {
         })
 
         // schedule domain verification in 30 seconds
-        await models.$executeRaw`INSERT INTO pgboss.job (name, data, startafter, keepuntil)
-          VALUES ('domainVerification',
-                  jsonb_build_object('domainId', ${updatedDomain.id}::INTEGER),
-                  now() + interval '30 seconds',
-                  now() + interval '2 days')`
+        await models.$executeRaw`
+        INSERT INTO pgboss.job (name, data, retrylimit, retrydelay, startafter, keepuntil)
+        VALUES ('domainVerification',
+                jsonb_build_object('domainId', ${updatedDomain.id}::INTEGER),
+                3,
+                30,
+                now() + interval '30 seconds',
+                now() + interval '2 days')`
 
         return updatedDomain
       } else {
         try {
+          // delete any existing domain verification jobs
+          await models.$queryRaw`
+          DELETE FROM pgboss.job
+          WHERE name = 'domainVerification'
+                AND data->>'domainId' = ${existing.id}::TEXT`
+
           return await models.customDomain.delete({ where: { subName } })
         } catch (error) {
           console.error(error)

lib/domain-verification.js

@@ -51,10 +51,14 @@ export async function certDetails (certificateArn) {
 // TODO: Test with real values, localstack don't have this info until the certificate is issued
 export async function getValidationValues (certificateArn) {
   const certificate = await certDetails(certificateArn)
-  console.log(certificate.DomainValidationOptions)
+  if (!certificate || !certificate.Certificate || !certificate.Certificate.DomainValidationOptions) {
+    return { cname: null, value: null }
+  }
+
+  console.log(certificate.Certificate.DomainValidationOptions)
   return {
-    cname: certificate.DomainValidationOptions[0].ResourceRecord.Name,
-    value: certificate.DomainValidationOptions[0].ResourceRecord.Value
+    cname: certificate.Certificate.DomainValidationOptions[0]?.ResourceRecord?.Name || null,
+    value: certificate.Certificate.DomainValidationOptions[0]?.ResourceRecord?.Value || null
   }
 }
 

worker/domainVerification.js

@@ -1,7 +1,7 @@
 import createPrisma from '@/lib/create-prisma'
-import { verifyDomainDNS, issueDomainCertificate, checkCertificateStatus, getValidationValues } from '@/lib/domain-verification'
+import { verifyDomainDNS, issueDomainCertificate, checkCertificateStatus, getValidationValues, deleteCertificate } from '@/lib/domain-verification'
 
-export async function domainVerification ({ data: { domainId }, boss }) {
+export async function domainVerification ({ id: jobId, data: { domainId }, boss }) {
   // establish connection to database
   const models = createPrisma({ connectionParams: { connection_limit: 1 } })
   try {
@@ -13,25 +13,44 @@ export async function domainVerification ({ data: { domainId }, boss }) {
     }
 
     // start verification process
-    const result = await verifyDomain(domain, boss)
+    const result = await verifyDomain(domain)
+    console.log(`domain verification result: ${JSON.stringify(result)}`)
 
     // update the domain with the result
     await models.customDomain.update({
       where: { id: domainId },
-      data: { ...domain, ...result }
+      data: result
     })
 
     // if the result is PENDING it means we still have to verify the domain
     // if it's not PENDING, we stop the verification process.
     if (result.status === 'PENDING') {
       // we still need to verify the domain, schedule the job to run again in 5 minutes
       const jobId = await boss.send('domainVerification', { domainId }, {
-        startAfter: 1000 * 60 * 5 // start the job after 5 minutes
+        startAfter: 60 * 5, // start the job after 5 minutes
+        retryLimit: 3,
+        retryDelay: 30 // on critical errors, retry every 5 minutes
       })
       console.log(`domain ${domain.domain} is still pending verification, created job with ID ${jobId} to run in 5 minutes`)
     }
   } catch (error) {
     console.error(`couldn't verify domain with ID ${domainId}: ${error.message}`)
+
+    // get the job details to get the retry count
+    const jobDetails = await boss.getJobById(jobId)
+    console.log(`job details: ${JSON.stringify(jobDetails)}`)
+    // if we couldn't verify the domain, put it on hold if it exists and delete any related verification jobs
+    if (jobDetails?.retrycount >= 3) {
+      console.log(`couldn't verify domain with ID ${domainId} for the third time, putting it on hold if it exists and deleting any related domain verification jobs`)
+      await models.customDomain.update({ where: { id: domainId }, data: { status: 'HOLD' } })
+      // delete any related domain verification jobs
+      await models.$queryRaw`
+      DELETE FROM pgboss.job
+      WHERE name = 'domainVerification'
+            AND data->>'domainId' = ${domainId}::TEXT`
+    }
+
+    throw error
   }
 }
 
@@ -67,7 +86,7 @@ async function verifyDomain (domain) {
   if (dnsState === 'VERIFIED' && !sslArn) {
     sslArn = await issueDomainCertificate(domain.domain)
     if (sslArn) {
-      console.log(`SSL certificate issued for ${domain.domain} with ARN ${sslArn}, will verify with ACM in 5 minutes`)
+      console.log(`SSL certificate issued for ${domain.domain} with ARN ${sslArn}, will verify with ACM`)
     } else {
       console.log(`couldn't issue SSL certificate for ${domain.domain}, will retry certificate issuance in 5 minutes`)
     }
@@ -78,9 +97,9 @@ async function verifyDomain (domain) {
   let acmValidationCname = verification.ssl.cname || null
   let acmValidationValue = verification.ssl.value || null
   if (sslArn && !acmValidationCname && !acmValidationValue) {
-    const { cname, value } = await getValidationValues(sslArn)
-    acmValidationCname = cname
-    acmValidationValue = value
+    const values = await getValidationValues(sslArn)
+    acmValidationCname = values?.cname || null
+    acmValidationValue = values?.value || null
     if (acmValidationCname && acmValidationValue) {
       console.log(`Validation values retrieved for ${domain.domain}, will check ACM validation status`)
     } else {
@@ -91,7 +110,7 @@ async function verifyDomain (domain) {
   // step 4: check if the certificate is validated by ACM
   // if DNS is verified and we have a SSL certificate
   // it can happen that we just issued the certificate and it's not yet validated by ACM
-  if (dnsState === 'VERIFIED' && sslArn && sslState !== 'VERIFIED') {
+  if (sslArn && sslState !== 'VERIFIED') {
     sslState = await checkCertificateStatus(sslArn)
     switch (sslState) {
       case 'VERIFIED':
@@ -114,13 +133,20 @@ async function verifyDomain (domain) {
   // if the domain has failed in some way and it's been 48 hours, put it on hold
   if (status !== 'ACTIVE' && domain.createdAt < new Date(Date.now() - 1000 * 60 * 60 * 24 * 2)) {
     status = 'HOLD'
+    // we stopped domain verification, delete the certificate as it will expire after 72 hours anyway
+    if (sslArn) {
+      console.log(`domain ${domain.domain} is on hold, deleting certificate as it will expire after 72 hours`)
+      const result = await deleteCertificate(sslArn)
+      console.log(`delete certificate attempt for ${domain.domain}, result: ${JSON.stringify(result)}`)
+    }
   }
 
   return {
     lastVerifiedAt,
     status,
     verification: {
       dns: {
+        ...verification.dns,
         state: dnsState
       },
       ssl: {