import fs from 'fs'
const all=JSON.parse(fs.readFileSync(process.argv[2],'utf8'))
const norm=s=>s.toLowerCase().replace(/[^a-z0-9 ]/g,' ').replace(/\s+/g,' ').trim()
const seen=new Set(), jobs=[]
for (const j of all.filter(x=>/\bsoftware engineer\b/i.test(x.title))) {
  const k=j.co+'|'+norm(j.title)+'|'+j.content.length
  if(!seen.has(k)){seen.add(k);jobs.push(j)}
}
const unesc=s=>s.replace(/&lt;/g,'<').replace(/&gt;/g,'>').replace(/&amp;/g,'&').replace(/&quot;/g,'"').replace(/&#39;/g,"'").replace(/&nbsp;/g,' ')
const texts=jobs.map(j=>unesc(j.content).replace(/<[^>]+>/g,' ').replace(/\s+/g,' '))

const VERB=`to|above|beyond|the|through|into|out|back|live|forward|deep|hand|and|for|with|on|in|a|an|from|over|straight|hard|far|wrong`
// Ambiguous English words: case-sensitive, plus exclusions. Everything else: case-insensitive.
const T = [
 ['Languages','Python',/\bPython\b/i],['Languages','Java',/\bJava\b(?!Script)/i],
 ['Languages','JavaScript',/\bJavaScript\b/i],['Languages','TypeScript',/\bTypeScript\b/i],
 ['Languages','Go',new RegExp(`\\bGolang\\b|\\bGo\\b(?!\\s+(?:${VERB})\\b)`)],
 ['Languages','C++',/C\+\+/],['Languages','C#',/C#/],['Languages','Ruby',/\bRuby\b/i],
 ['Languages','Rust',/\bRust\b/],['Languages','Scala',/\bScala\b/i],['Languages','Kotlin',/\bKotlin\b/i],
 ['Languages','Swift',/\bSwift\b(?!\s*(?:ly|er))/],['Languages','SQL',/\bSQL\b/i],
 ['Cloud & infra','AWS',/\bAWS\b/i],['Cloud & infra','GCP / Google Cloud',/\bGCP\b|\bGoogle Cloud\b/i],
 ['Cloud & infra','Azure',/\bAzure\b/i],['Cloud & infra','Kubernetes',/\bKubernetes\b|\bK8s\b/i],
 ['Cloud & infra','Docker',/\bDocker\b/i],['Cloud & infra','Terraform',/\bTerraform\b/i],
 ['Cloud & infra','Linux',/\bLinux\b/i],['Cloud & infra','microservices',/\bmicro-?services\b/i],
 ['Data','PostgreSQL',/\bPostgre(?:SQL)?\b|\bPostgres\b/i],['Data','MySQL',/\bMySQL\b/i],
 ['Data','MongoDB',/\bMongoDB\b/i],['Data','Redis',/\bRedis\b/i],['Data','Kafka',/\bKafka\b/i],
 ['Data','Snowflake',/\bSnowflake\b/i],['Data','Spark',/\bApache Spark\b|\bSpark\b(?!\s*(?:ing|ed|s\b))/],
 ['Data','Elasticsearch',/\bElastic-?search\b/i],
 ['Frontend','React',/\bReact\b(?!\s*(?:to|ing|ed|s\b))/],['Frontend','Angular',/\bAngular\b/i],
 ['Frontend','Vue',/\bVue(?:\.js)?\b/i],['Frontend','Node.js',/\bNode\.?js\b/i],
 ['Frontend','GraphQL',/\bGraphQL\b/i],['Frontend','REST / RESTful',/\bREST(?:ful)?\b/],
 ['Practice','distributed systems',/\bdistributed systems\b/i],['Practice','CI/CD',/\bCI\s*\/\s*CD\b|\bcontinuous (?:integration|delivery)\b/i],
 ['Practice','Agile / Scrum',/\bagile\b|\bscrum\b/i],['Practice','code review',/\bcode review/i],
 ['Practice','on-call',/\bon-?call\b/i],['Practice','mentoring',/\bmentor/i],
 ['Practice','cross-functional',/\bcross-?functional\b/i],['Practice','scalability',/\bscalab/i],
 ['Practice','unit / automated testing',/\bunit test|\bautomated test/i],
 ['Practice','system design / architecture',/\bsystem design\b|\barchitecture\b/i],
]
const rows=T.map(([cat,term,re])=>{
  const n=texts.filter(t=>re.test(t)).length
  return {cat,term,n,pct:+(100*n/texts.length).toFixed(1)}
}).sort((a,b)=>b.n-a.n)
const meta={role:'Software Engineer',n:jobs.length,companies:[...new Set(jobs.map(j=>j.co))].sort(),
  collected:new Date(Date.now() - new Date().getTimezoneOffset() * 60000).toISOString().slice(0,10),
  medianWords:texts.map(t=>t.split(' ').length).sort((a,b)=>a-b)[Math.floor(texts.length/2)]}
fs.writeFileSync(process.argv[3],JSON.stringify({...meta,rows},null,1))
console.log(`n=${meta.n} postings, ${meta.companies.length} companies, median ${meta.medianWords} words`)
console.log('\nCORRECTED TOP 25')
rows.slice(0,25).forEach((r,i)=>console.log(`  ${String(i+1).padStart(2)}. ${r.term.padEnd(28)} ${String(r.pct).padStart(5)}%  (${r.n})`))