import fs from 'fs'
const all = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'))
const norm = s => s.toLowerCase().replace(/[^a-z0-9 ]/g, ' ').replace(/\s+/g, ' ').trim()

// Cyber roles only. The exclusion list matters: "Lead Physical Security Engineer"
// matches the include pattern and is a different profession.
const INCLUDE = /\b(security engineer|security analyst|security architect|application security|product security|infrastructure security|cloud security|offensive security|detection (and response )?engineer|incident response|penetration test|red team|threat (intel|detection|hunting)|appsec|infosec|cybersecurity|security operations|security researcher|security compliance|security software engineer)\b/i
const EXCLUDE = /\bphysical security\b|\bsecurity guard\b|\bcampus\b|\bsecurity officer\b|\bexecutive protection\b/i

const seen = new Set(), jobs = []
for (const j of all.filter(x => INCLUDE.test(x.title) && !EXCLUDE.test(x.title))) {
  const k = j.co + '|' + norm(j.title) + '|' + j.content.length
  if (!seen.has(k)) { seen.add(k); jobs.push(j) }
}
const unesc = s => s.replace(/&lt;/g,'<').replace(/&gt;/g,'>').replace(/&amp;/g,'&')
  .replace(/&quot;/g,'"').replace(/&#39;/g,"'").replace(/&nbsp;/g,' ')
const texts = jobs.map(j => unesc(j.content).replace(/<[^>]+>/g,' ').replace(/\s+/g,' '))

const VERB = `to|above|beyond|the|through|into|out|back|live|forward|deep|hand|and|for|with|on|in|a|an|from|over`
const T = [
 ['Certifications','CISSP',/\bCISSP\b/i],['Certifications','OSCP',/\bOSCP\b/i],
 ['Certifications','CISM',/\bCISM\b/i],['Certifications','CEH',/\bCEH\b/],
 ['Certifications','Security+',/\bSecurity\s*\+/i],['Certifications','GIAC / SANS',/\bGIAC\b|\bSANS\b/i],
 ['Frameworks & compliance','SOC 2',/\bSOC\s*2\b/i],['Frameworks & compliance','ISO 27001',/\bISO\s*27001\b/i],
 ['Frameworks & compliance','NIST',/\bNIST\b/i],['Frameworks & compliance','PCI DSS',/\bPCI[\s-]?DSS\b|\bPCI\b/i],
 ['Frameworks & compliance','GDPR',/\bGDPR\b/i],['Frameworks & compliance','HIPAA',/\bHIPAA\b/i],
 ['Frameworks & compliance','FedRAMP',/\bFedRAMP\b/i],['Frameworks & compliance','MITRE ATT&CK',/\bMITRE\b|\bATT&CK\b|\bATT&amp;CK\b/i],
 ['Cloud & infra','AWS',/\bAWS\b/i],['Cloud & infra','GCP / Google Cloud',/\bGCP\b|\bGoogle Cloud\b/i],
 ['Cloud & infra','Azure',/\bAzure\b/i],['Cloud & infra','Kubernetes',/\bKubernetes\b|\bK8s\b/i],
 ['Cloud & infra','Terraform',/\bTerraform\b/i],['Cloud & infra','Linux',/\bLinux\b/i],
 ['Tools','SIEM',/\bSIEM\b/i],['Tools','EDR / XDR',/\bEDR\b|\bXDR\b/i],['Tools','Splunk',/\bSplunk\b/i],
 ['Tools','Burp Suite',/\bBurp\b/i],['Tools','SAST / DAST',/\bSAST\b|\bDAST\b/i],
 ['Tools','vulnerability scanning',/\bNessus\b|\bQualys\b|\bvulnerability scan/i],
 ['Practice','threat modeling',/\bthreat model/i],['Practice','incident response',/\bincident response\b/i],
 ['Practice','penetration testing',/\bpen(etration)?[\s-]?test/i],['Practice','vulnerability management',/\bvulnerability management\b/i],
 ['Practice','IAM / identity',/\bIAM\b|\bidentity and access\b/i],['Practice','zero trust',/\bzero[\s-]trust\b/i],
 ['Practice','detection engineering',/\bdetection engineering\b|\bdetection rules?\b/i],
 ['Practice','cryptography',/\bcryptograph|\bencryption\b/i],['Practice','secure code review',/\bcode review\b/i],
 ['Practice','red team',/\bred team/i],['Practice','on-call',/\bon-?call\b/i],
 ['Languages','Python',/\bPython\b/i],['Languages','Go',new RegExp(`\\bGolang\\b|\\bGo\\b(?!\\s+(?:${VERB})\\b)`)],
 ['Languages','Bash / shell',/\bBash\b|\bshell scripting\b/i],
]
const n = texts.length
const moe = p => +(1.96 * Math.sqrt((p/100)*(1-p/100)/n) * 100).toFixed(1)
const rows = T.map(([cat, term, re]) => {
  const c = texts.filter(t => re.test(t)).length
  const pct = +(100*c/n).toFixed(1)
  return { cat, term, n: c, pct, moe: moe(pct) }
}).sort((a,b) => b.n - a.n)
const out = { role: 'Cybersecurity', n, companies: [...new Set(jobs.map(j=>j.co))].sort(),
  collected: new Date(Date.now() - new Date().getTimezoneOffset() * 60000).toISOString().slice(0,10),
  medianWords: texts.map(t=>t.split(' ').length).sort((a,b)=>a-b)[Math.floor(n/2)], rows }
fs.writeFileSync(process.argv[3], JSON.stringify(out, null, 1))
console.log(`n=${n} from ${out.companies.length} companies, median ${out.medianWords} words`)
rows.slice(0,18).forEach((r,i)=>console.log(`  ${String(i+1).padStart(2)}. ${r.term.padEnd(26)} ${String(r.pct).padStart(5)}% ±${r.moe}`))