import fs from 'fs'
const all = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'))
const norm = s => s.toLowerCase().replace(/[^a-z0-9 ]/g, ' ').replace(/\s+/g, ' ').trim()

// Individual contributors only. "Manager, Site Reliability Engineering" and
// "Staff Technical Program Manager, Site Reliability Engineering" both match the
// include pattern and ask for different things than the engineers they manage.
// Excluding them costs 12 postings and keeps the sample one job.
const INCLUDE = /\b(devops|dev ops|site reliability|sre|platform engineer|infrastructure engineer)\b/i
const EXCLUDE = /\bintern\b|\binternship\b|skillbridge|\bprogram manager\b|\bmanager\b|\bdirector\b|\bhead of\b|\bvp\b/i

const seen = new Set(), jobs = []
for (const j of all.filter(x => INCLUDE.test(x.title) && !EXCLUDE.test(x.title))) {
  const k = j.co + '|' + norm(j.title) + '|' + j.content.length
  if (!seen.has(k)) { seen.add(k); jobs.push(j) }
}
const unesc = s => s.replace(/&lt;/g,'<').replace(/&gt;/g,'>').replace(/&amp;/g,'&')
  .replace(/&quot;/g,'"').replace(/&#39;/g,"'").replace(/&nbsp;/g,' ')
const texts = jobs.map(j => unesc(j.content).replace(/<[^>]+>/g,' ').replace(/\s+/g,' '))

// "Go" needs the same negative lookahead the engineering study uses: a bare
// \bGo\b also matches "go to market" and "go above and beyond", and overstates
// it here by 4.5 points (51.5% against 47.0%). Every other term in this list was
// checked loose-vs-strict and came back identical, including Helm — no "at the
// helm" contamination in this corpus.
const VERB = `to|above|beyond|the|through|into|out|back|live|forward|deep|hand|and|for|with|on|in|a|an|from|over`
const T = [
 ['Cloud & infrastructure','AWS',/\bAWS\b|\bAmazon Web Services\b/i],
 ['Cloud & infrastructure','Kubernetes',/\bKubernetes\b|\bK8s\b/i],
 ['Cloud & infrastructure','GCP / Google Cloud',/\bGCP\b|\bGoogle Cloud\b/i],
 ['Cloud & infrastructure','Azure',/\bAzure\b/i],
 ['Cloud & infrastructure','Docker / containers',/\bDocker\b|\bcontaineri[sz]/i],
 ['Cloud & infrastructure','Linux',/\bLinux\b/i],
 ['Cloud & infrastructure','networking',/\bnetworking\b|\bTCP\/IP\b|\bDNS\b/i],
 ['Cloud & infrastructure','serverless',/\bserverless\b|\bLambda\b/i],

 ['Provisioning & delivery','Terraform',/\bTerraform\b/i],
 ['Provisioning & delivery','infrastructure as code',/\binfrastructure[- ]as[- ]code\b|\bIaC\b/i],
 ['Provisioning & delivery','CI/CD',/\bCI\/CD\b|\bcontinuous (integration|delivery|deployment)\b/i],
 ['Provisioning & delivery','Helm',/\bHelm\b/],
 ['Provisioning & delivery','ArgoCD',/\bArgo\s?CD\b/i],
 ['Provisioning & delivery','GitHub Actions',/\bGitHub Actions\b/i],
 ['Provisioning & delivery','Jenkins',/\bJenkins\b/i],
 ['Provisioning & delivery','GitLab',/\bGitLab\b/i],
 ['Provisioning & delivery','Ansible',/\bAnsible\b/i],
 ['Provisioning & delivery','Chef',/\bChef\b/],
 ['Provisioning & delivery','Puppet',/\bPuppet\b/],

 ['Observability','observability',/\bobservabilit/i],
 ['Observability','monitoring',/\bmonitoring\b/i],
 ['Observability','Prometheus',/\bPrometheus\b/i],
 ['Observability','Grafana',/\bGrafana\b/i],
 ['Observability','Datadog',/\bDatadog\b/i],
 ['Observability','logging',/\blogging\b|\blog aggregation\b/i],
 ['Observability','distributed tracing',/\btracing\b|\bOpenTelemetry\b/i],
 ['Observability','PagerDuty',/\bPagerDuty\b/i],

 ['Reliability practice','on-call',/\bon-?call\b/i],
 ['Reliability practice','incident response',/\bincident (response|management)\b/i],
 ['Reliability practice','SLO / SLA',/\bSLOs?\b|\bSLAs?\b|\bSLIs?\b/],
 ['Reliability practice','postmortems',/\bpost[- ]?mortem/i],
 ['Reliability practice','automation',/\bautomat/i],
 ['Reliability practice','scalability',/\bscalab|\bscaling\b/i],
 ['Reliability practice','toil reduction',/\btoil\b/i],
 ['Reliability practice','capacity planning',/\bcapacity planning\b/i],
 ['Reliability practice','disaster recovery',/\bdisaster recovery\b|\bbusiness continuity\b/i],
 ['Reliability practice','chaos engineering',/\bchaos engineering\b|\bchaos testing\b/i],
 ['Reliability practice','cost optimization',/\bcost optimi[sz]|\bFinOps\b/i],

 ['Languages','Python',/\bPython\b/i],
 ['Languages','Go',new RegExp(`\\bGolang\\b|\\bGo\\b(?!\\s+(?:${VERB})\\b)`)],
 ['Languages','Bash / shell',/\bBash\b|\bshell scripting\b/i],
 ['Languages','Rust',/\bRust\b/],
]
const n = texts.length
const moe = p => +(1.96 * Math.sqrt((p/100)*(1-p/100)/n) * 100).toFixed(1)
const rows = T.map(([cat, term, re]) => {
  const c = texts.filter(t => re.test(t)).length
  const pct = +(100*c/n).toFixed(1)
  return { cat, term, n: c, pct, moe: moe(pct) }
}).sort((a,b) => b.n - a.n)
const out = { role: 'DevOps / SRE / Platform Engineer', n, companies: [...new Set(jobs.map(j=>j.co))].sort(),
  collected: new Date(Date.now() - new Date().getTimezoneOffset() * 60000).toISOString().slice(0,10),
  medianWords: texts.map(t=>t.split(' ').length).sort((a,b)=>a-b)[Math.floor(n/2)], rows }
fs.writeFileSync(process.argv[3], JSON.stringify(out, null, 1))
console.log(`n=${n} from ${out.companies.length} companies, median ${out.medianWords} words`)
rows.forEach((r,i)=>console.log(`  ${String(i+1).padStart(2)}. ${r.term.padEnd(24)} ${String(r.pct).padStart(5)}% ±${r.moe}  (${r.n})`))