[{"data":1,"prerenderedAt":1560},["ShallowReactive",2],{"navigation_docs":3,"-engineering-contributing-database-database-refresh":449,"-engineering-contributing-database-database-refresh-surround":1555},[4,8,72,102,248,294,331,352,445],{"title":5,"path":6,"stem":7},"Introduction","\u002Fintroduction","0.introduction",{"title":9,"icon":10,"path":11,"stem":12,"children":13,"page":67},"Company","i-lucide-building-2","\u002Fcompany","1.company",[14,18,22,26,30,34,38,42,46,50,54,68],{"title":15,"path":16,"stem":17},"About","\u002Fcompany\u002Fabout","1.company\u002F0.about",{"title":19,"path":20,"stem":21},"Values","\u002Fcompany\u002Fvalues","1.company\u002F1.values",{"title":23,"path":24,"stem":25},"Communication","\u002Fcompany\u002Fcommunication","1.company\u002Fcommunication",{"title":27,"path":28,"stem":29},"Competition","\u002Fcompany\u002Fcompetition","1.company\u002Fcompetition",{"title":31,"path":32,"stem":33},"Hybrid Working","\u002Fcompany\u002Fhybrid-working","1.company\u002Fhybrid-working",{"title":35,"path":36,"stem":37},"Manchester Office","\u002Fcompany\u002Foffice","1.company\u002Foffice",{"title":39,"path":40,"stem":41},"Operations","\u002Fcompany\u002Foperations","1.company\u002Foperations",{"title":43,"path":44,"stem":45},"Policies","\u002Fcompany\u002Fpolicies","1.company\u002Fpolicies",{"title":47,"path":48,"stem":49},"Problem Statement","\u002Fcompany\u002Fproblem-statement","1.company\u002Fproblem-statement",{"title":51,"path":52,"stem":53},"Product Strategy","\u002Fcompany\u002Fproduct-strategy","1.company\u002Fproduct-strategy",{"title":55,"path":56,"stem":57,"children":58,"page":67},"Products","\u002Fcompany\u002Fproducts","1.company\u002Fproducts",[59,63],{"title":60,"path":61,"stem":62},"Capability Exchange","\u002Fcompany\u002Fproducts\u002Fcapability-exchange","1.company\u002Fproducts\u002Fcapability-exchange",{"title":64,"path":65,"stem":66},"ESProfiler Platform","\u002Fcompany\u002Fproducts\u002Fesprofiler","1.company\u002Fproducts\u002Fesprofiler",false,{"title":69,"path":70,"stem":71},"Security","\u002Fcompany\u002Fsecurity","1.company\u002Fsecurity",{"title":73,"icon":74,"path":75,"stem":76,"children":77,"page":67},"People Ops","i-lucide-users","\u002Fpeople-ops","2.people-ops",[78,82,86,90,94,98],{"title":79,"path":80,"stem":81},"Compensation","\u002Fpeople-ops\u002Fcompensation","2.people-ops\u002Fcompensation",{"title":83,"path":84,"stem":85},"Education","\u002Fpeople-ops\u002Feducation","2.people-ops\u002Feducation",{"title":87,"path":88,"stem":89},"Expenses","\u002Fpeople-ops\u002Fexpenses","2.people-ops\u002Fexpenses",{"title":91,"path":92,"stem":93},"Holiday & Leave","\u002Fpeople-ops\u002Fleave","2.people-ops\u002Fleave",{"title":95,"path":96,"stem":97},"Onboarding","\u002Fpeople-ops\u002Fonboarding","2.people-ops\u002Fonboarding",{"title":99,"path":100,"stem":101},"Recruitment","\u002Fpeople-ops\u002Frecruitment","2.people-ops\u002Frecruitment",{"title":103,"icon":104,"path":105,"stem":106,"children":107,"page":67},"Engineering","i-lucide-rocket","\u002Fengineering","3.engineering",[108,168,189,210,232,240,244],{"title":109,"path":110,"stem":111,"children":112,"page":67},"Contributing","\u002Fengineering\u002Fcontributing","3.engineering\u002Fcontributing",[113,117,121,125,129,133,146,155],{"title":114,"path":115,"stem":116},"Development Setup","\u002Fengineering\u002Fcontributing\u002Fdevelopment-setup","3.engineering\u002Fcontributing\u002F1.development-setup",{"title":118,"path":119,"stem":120},"Engineering Operations","\u002Fengineering\u002Fcontributing\u002Fengineering-operations","3.engineering\u002Fcontributing\u002F2.engineering-operations",{"title":122,"path":123,"stem":124},"Documentation","\u002Fengineering\u002Fcontributing\u002Fdocumentation","3.engineering\u002Fcontributing\u002F3.documentation",{"title":126,"path":127,"stem":128},"Agentic Coding","\u002Fengineering\u002Fcontributing\u002Fagentic-coding","3.engineering\u002Fcontributing\u002F4.agentic-coding",{"title":130,"path":131,"stem":132},"Coding Standards","\u002Fengineering\u002Fcontributing\u002Fcoding-standards","3.engineering\u002Fcontributing\u002F5.coding-standards",{"title":134,"path":135,"stem":136,"children":137,"page":67},"Back End","\u002Fengineering\u002Fcontributing\u002Fback-end","3.engineering\u002Fcontributing\u002Fback-end",[138,142],{"title":139,"path":140,"stem":141},"API Guidelines","\u002Fengineering\u002Fcontributing\u002Fback-end\u002Fapi-guidelines","3.engineering\u002Fcontributing\u002Fback-end\u002Fapi-guidelines",{"title":143,"path":144,"stem":145},"LLM Prompts & Langfuse Integration","\u002Fengineering\u002Fcontributing\u002Fback-end\u002Fllm-prompts","3.engineering\u002Fcontributing\u002Fback-end\u002Fllm-prompts",{"title":147,"path":148,"stem":149,"children":150,"page":67},"Database","\u002Fengineering\u002Fcontributing\u002Fdatabase","3.engineering\u002Fcontributing\u002Fdatabase",[151],{"title":152,"path":153,"stem":154},"Database Volumes","\u002Fengineering\u002Fcontributing\u002Fdatabase\u002Fdatabase-refresh","3.engineering\u002Fcontributing\u002Fdatabase\u002Fdatabase-refresh",{"title":156,"path":157,"stem":158,"children":159,"page":67},"Front End","\u002Fengineering\u002Fcontributing\u002Ffront-end","3.engineering\u002Fcontributing\u002Ffront-end",[160,164],{"title":161,"path":162,"stem":163},"Data Attributes","\u002Fengineering\u002Fcontributing\u002Ffront-end\u002Fdata-attributes","3.engineering\u002Fcontributing\u002Ffront-end\u002Fdata-attributes",{"title":165,"path":166,"stem":167},"Testing","\u002Fengineering\u002Fcontributing\u002Ffront-end\u002Ftesting","3.engineering\u002Fcontributing\u002Ffront-end\u002Ftesting",{"title":169,"path":170,"stem":171,"children":172},"Deployment","\u002Fengineering\u002Fdeployment","3.engineering\u002Fdeployment\u002F0.index",[173,174,177,181,185],{"title":169,"path":170,"stem":171},{"title":60,"path":175,"stem":176},"\u002Fengineering\u002Fdeployment\u002Fcapability-exchange","3.engineering\u002Fdeployment\u002Fcapability-exchange",{"title":178,"path":179,"stem":180},"Langfuse Deployment","\u002Fengineering\u002Fdeployment\u002Fecs-langfuse-deployment","3.engineering\u002Fdeployment\u002Fecs-langfuse-deployment",{"title":182,"path":183,"stem":184},"ESP Platform Configuration","\u002Fengineering\u002Fdeployment\u002Fesp-platform-configuration","3.engineering\u002Fdeployment\u002Fesp-platform-configuration",{"title":186,"path":187,"stem":188},"Platform","\u002Fengineering\u002Fdeployment\u002Fplatform","3.engineering\u002Fdeployment\u002Fplatform",{"title":190,"path":191,"stem":192,"children":193,"page":67},"Github","\u002Fengineering\u002Fgithub","3.engineering\u002Fgithub",[194,198,202,206],{"title":195,"path":196,"stem":197},"Packages","\u002Fengineering\u002Fgithub\u002Fpackages","3.engineering\u002Fgithub\u002Fpackages",{"title":199,"path":200,"stem":201},"Personal Access Token","\u002Fengineering\u002Fgithub\u002Fpersonal-access-token","3.engineering\u002Fgithub\u002Fpersonal-access-token",{"title":203,"path":204,"stem":205},"Troubleshooting","\u002Fengineering\u002Fgithub\u002Ftroubleshooting","3.engineering\u002Fgithub\u002Ftroubleshooting",{"title":207,"path":208,"stem":209},"Workflows","\u002Fengineering\u002Fgithub\u002Fworkflows","3.engineering\u002Fgithub\u002Fworkflows",{"title":211,"path":212,"stem":213,"children":214},"Platform Ops","\u002Fengineering\u002Fplatform-ops","3.engineering\u002Fplatform-ops\u002F0.index",[215,216,220,224,228],{"title":211,"path":212,"stem":213},{"title":217,"path":218,"stem":219},"Production Database","\u002Fengineering\u002Fplatform-ops\u002Fdatabase-connection","3.engineering\u002Fplatform-ops\u002Fdatabase-connection",{"title":221,"path":222,"stem":223},"New Tenant Creation","\u002Fengineering\u002Fplatform-ops\u002Fnew-tenant-creation","3.engineering\u002Fplatform-ops\u002Fnew-tenant-creation",{"title":225,"path":226,"stem":227},"Releases","\u002Fengineering\u002Fplatform-ops\u002Frelease","3.engineering\u002Fplatform-ops\u002Frelease",{"title":229,"path":230,"stem":231},"Versioning","\u002Fengineering\u002Fplatform-ops\u002Fversioning","3.engineering\u002Fplatform-ops\u002Fversioning",{"title":186,"path":233,"stem":234,"children":235,"page":67},"\u002Fengineering\u002Fplatform","3.engineering\u002Fplatform",[236],{"title":237,"path":238,"stem":239},"useAPI","\u002Fengineering\u002Fplatform\u002Fuse-api","3.engineering\u002Fplatform\u002Fuse-api",{"title":241,"path":242,"stem":243},"Project Management","\u002Fengineering\u002Fproject-management","3.engineering\u002Fproject-management",{"title":245,"path":246,"stem":247},"Tools","\u002Fengineering\u002Ftools","3.engineering\u002Ftools",{"title":249,"icon":250,"path":251,"stem":252,"children":253,"page":67},"Design","i-lucide-palette","\u002Fdesign","4.design",[254,275,279,283,287,290],{"title":255,"path":256,"stem":257,"children":258,"page":67},"Personas","\u002Fdesign\u002Fpersonas","4.design\u002F0.personas",[259,263,267,271],{"title":260,"path":261,"stem":262},"CISO - Simon","\u002Fdesign\u002Fpersonas\u002F01-ciso-simon","4.design\u002F0.personas\u002F01-ciso-simon",{"title":264,"path":265,"stem":266},"Head of Cyber - Harry","\u002Fdesign\u002Fpersonas\u002F02-head-of-cyber-harry","4.design\u002F0.personas\u002F02-head-of-cyber-harry",{"title":268,"path":269,"stem":270},"Security Architect - Sasha","\u002Fdesign\u002Fpersonas\u002F03-security-architect-sasha","4.design\u002F0.personas\u002F03-security-architect-sasha",{"title":272,"path":273,"stem":274},"Procurement Officer - Paige","\u002Fdesign\u002Fpersonas\u002F04-procurement-officer-paige","4.design\u002F0.personas\u002F04-procurement-officer-paige",{"title":276,"path":277,"stem":278},"Design Thinking","\u002Fdesign\u002Fdesign-thinking","4.design\u002F1.design-thinking",{"title":280,"path":281,"stem":282},"Design & Development","\u002Fdesign\u002Fdesign-and-development","4.design\u002F3.design-and-development",{"title":284,"path":285,"stem":286},"Branding","\u002Fdesign\u002Fbranding","4.design\u002F4.branding",{"title":245,"path":288,"stem":289},"\u002Fdesign\u002Ftools","4.design\u002F5.tools",{"title":291,"path":292,"stem":293},"Customer Success","\u002Fdesign\u002Fworking-with-customers","4.design\u002F6.working-with-customers",{"title":295,"icon":296,"path":297,"stem":298,"children":299,"page":67},"Sales","i-lucide-dollar-sign","\u002Fsales","4.sales",[300,320,324,328],{"title":301,"path":302,"stem":303,"children":304,"page":67},"Process","\u002Fsales\u002Fprocess","4.sales\u002F0.process",[305,309,313,317],{"title":306,"path":307,"stem":308},"Lead","\u002Fsales\u002Fprocess\u002Flead","4.sales\u002F0.process\u002F0.lead",{"title":310,"path":311,"stem":312},"Deal Qualification","\u002Fsales\u002Fprocess\u002Fqualification","4.sales\u002F0.process\u002F1.qualification",{"title":314,"path":315,"stem":316},"Close","\u002Fsales\u002Fprocess\u002Fclose","4.sales\u002F0.process\u002F2.close",{"title":95,"path":318,"stem":319},"\u002Fsales\u002Fprocess\u002Fonboarding","4.sales\u002F0.process\u002Fonboarding",{"title":321,"path":322,"stem":323},"Meetings","\u002Fsales\u002Fmeetings","4.sales\u002Fmeetings",{"title":325,"path":326,"stem":327},"Metrics","\u002Fsales\u002Fmetrics","4.sales\u002Fmetrics",{"title":245,"path":329,"stem":330},"\u002Fsales\u002Ftools","4.sales\u002Ftools",{"title":332,"icon":333,"path":334,"stem":335,"children":336,"page":67},"Marketing","i-lucide-book-image","\u002Fmarketing","5.marketing",[337,341,345,348],{"title":338,"path":339,"stem":340},"Content","\u002Fmarketing\u002Fcontent","5.marketing\u002Fcontent",{"title":342,"path":343,"stem":344},"Messaging","\u002Fmarketing\u002Fmessaging","5.marketing\u002Fmessaging",{"title":245,"path":346,"stem":347},"\u002Fmarketing\u002Ftools","5.marketing\u002Ftools",{"title":349,"path":350,"stem":351},"Website","\u002Fmarketing\u002Fwebsite","5.marketing\u002Fwebsite",{"title":353,"icon":354,"path":355,"stem":356,"children":357,"page":67},"AI & Data Ops","i-lucide-database","\u002Fdata-ops","6.data-ops",[358,362,370,395,416,420],{"title":359,"path":360,"stem":361},"Account Portal (CAS)","\u002Fdata-ops\u002Faccount-portal","6.data-ops\u002Faccount-portal",{"title":60,"path":363,"stem":364,"children":365,"page":67},"\u002Fdata-ops\u002Fcapability-exchange","6.data-ops\u002Fcapability-exchange",[366],{"title":367,"path":368,"stem":369},"Leaderboard Calculation","\u002Fdata-ops\u002Fcapability-exchange\u002Fleaderboard-calculation","6.data-ops\u002Fcapability-exchange\u002Fleaderboard-calculation",{"title":371,"path":372,"stem":373,"children":374,"page":67},"Data Management","\u002Fdata-ops\u002Fdata-management","6.data-ops\u002Fdata-management",[375,379,383,387,391],{"title":376,"path":377,"stem":378},"Adding Products","\u002Fdata-ops\u002Fdata-management\u002Fadding-products","6.data-ops\u002Fdata-management\u002Fadding-products",{"title":380,"path":381,"stem":382},"Adding Vendors","\u002Fdata-ops\u002Fdata-management\u002Fadding-vendors","6.data-ops\u002Fdata-management\u002Fadding-vendors",{"title":384,"path":385,"stem":386},"Framework Mapping","\u002Fdata-ops\u002Fdata-management\u002Fframework-mapping","6.data-ops\u002Fdata-management\u002Fframework-mapping",{"title":388,"path":389,"stem":390},"Refreshing Vendors","\u002Fdata-ops\u002Fdata-management\u002Frefreshing-vendors","6.data-ops\u002Fdata-management\u002Frefreshing-vendors",{"title":392,"path":393,"stem":394},"Reviewing Draft Vendors","\u002Fdata-ops\u002Fdata-management\u002Freviewing-draft-vendors","6.data-ops\u002Fdata-management\u002Freviewing-draft-vendors",{"title":396,"path":397,"stem":398,"children":399,"page":67},"LLM Ops","\u002Fdata-ops\u002Fllm-ops","6.data-ops\u002Fllm-ops",[400,404,408,412],{"title":401,"path":402,"stem":403},"Agents","\u002Fdata-ops\u002Fllm-ops\u002Fagents","6.data-ops\u002Fllm-ops\u002F1.agents",{"title":405,"path":406,"stem":407},"ESPi Architecture & Query Flow","\u002Fdata-ops\u002Fllm-ops\u002Fespi-architecture","6.data-ops\u002Fllm-ops\u002F2.espi-architecture",{"title":409,"path":410,"stem":411},"Evaluating Agents","\u002Fdata-ops\u002Fllm-ops\u002Fevaluations","6.data-ops\u002Fllm-ops\u002F3.evaluations",{"title":413,"path":414,"stem":415},"Benchmarking ESPi","\u002Fdata-ops\u002Fllm-ops\u002Fbenchmarking-espi","6.data-ops\u002Fllm-ops\u002F4.benchmarking-espi",{"title":417,"path":418,"stem":419},"Message Queues","\u002Fdata-ops\u002Fmessage-queues","6.data-ops\u002Fmessage-queues",{"title":421,"path":422,"stem":423,"children":424,"page":67},"Models","\u002Fdata-ops\u002Fmodels","6.data-ops\u002Fmodels",[425,429,433,437,441],{"title":426,"path":427,"stem":428},"Universal Framework Mapper Vision","\u002Fdata-ops\u002Fmodels\u002Funiversal-framework-mapper-vision","6.data-ops\u002Fmodels\u002F1.universal-framework-mapper-vision",{"title":430,"path":431,"stem":432},"Framework Description Model (FDM)","\u002Fdata-ops\u002Fmodels\u002Fframework-description-model","6.data-ops\u002Fmodels\u002F2.framework-description-model",{"title":434,"path":435,"stem":436},"Relationship Definition Model (RDM)","\u002Fdata-ops\u002Fmodels\u002Frelationship-definition-model","6.data-ops\u002Fmodels\u002F3.relationship-definition-model",{"title":438,"path":439,"stem":440},"Relationship Mapping Model (RMM)","\u002Fdata-ops\u002Fmodels\u002Frelationship-mapping-model","6.data-ops\u002Fmodels\u002F4.relationship-mapping-model",{"title":442,"path":443,"stem":444},"Platform Framework Matrix V2","\u002Fdata-ops\u002Fmodels\u002Fplatform-framework-matrix-v2","6.data-ops\u002Fmodels\u002F5.platform-framework-matrix-v2",{"title":446,"path":447,"stem":448},"Glossary","\u002Fglossary","glossary",{"id":450,"title":152,"body":451,"description":1548,"extension":1549,"links":1550,"meta":1551,"navigation":1552,"path":153,"seo":1553,"stem":154,"__hash__":1554},"docs\u002F3.engineering\u002Fcontributing\u002Fdatabase\u002Fdatabase-refresh.md",{"type":452,"value":453,"toc":1527},"minimark",[454,476,483,488,499,524,527,534,609,624,628,635,640,643,648,691,702,707,747,754,768,772,784,790,811,817,820,874,880,884,896,902,950,964,971,975,985,988,996,999,1003,1016,1025,1028,1046,1055,1108,1122,1168,1177,1191,1202,1218,1221,1225,1229,1232,1238,1249,1256,1260,1277,1284,1291,1312,1322,1329,1333,1336,1344,1347,1358,1368,1372,1379,1390,1397,1401,1408,1433,1450,1461,1496,1500,1515,1523],[455,456,457,458,462,463,467,468,475],"p",{},"We don't import production dumps individually. One person refreshes the data and publishes it as a ",[459,460,461],"strong",{},"volume image"," to ",[464,465,466],"code",{},"ghcr.io",", and everyone else pulls it through ",[469,470,474],"a",{"href":471,"rel":472},"https:\u002F\u002Fgithub.com\u002FES-Profiler\u002Finfra-devenv-pro",[473],"nofollow","Dev Envi Pro",". That way the whole team works from the same dataset and only one person needs production access.",[455,477,478,479,482],{},"Each volume is published at four sizes, called ",[459,480,481],{},"tiers",".",[484,485,487],"h2",{"id":486},"using-a-volume","Using a volume",[455,489,490,491,494,495,498],{},"This is what most people do. From your ",[464,492,493],{},"infra-devenv-pro"," checkout, double click ",[464,496,497],{},"esprofiler-dev.bat"," or run it from a terminal:",[500,501,506],"pre",{"className":502,"code":503,"language":504,"meta":505,"style":505},"language-bash shiki shiki-themes material-theme-lighter material-theme material-theme-palenight",".\\esprofiler-dev.bat\n","bash","",[464,507,508],{"__ignoreMap":505},[509,510,513,516,520],"span",{"class":511,"line":512},"line",1,[509,514,482],{"class":515},"s2Zo4",[509,517,519],{"class":518},"sTEyZ","\\",[509,521,523],{"class":522},"sfazB","esprofiler-dev.bat\n",[455,525,526],{},"The launcher checks you have Python, creates a local virtual environment the first time, installs the dependencies and then opens the menu. It's safe to run from anywhere — it always works from its own folder.",[455,528,529,530,533],{},"Choose ",[464,531,532],{},"Import Volume",", then the volume and the tier you want. It pulls the image, swaps the volume contents and restarts the container. If the pull fails, nothing is touched.",[535,536,537,553],"table",{},[538,539,540],"thead",{},[541,542,543,547,550],"tr",{},[544,545,546],"th",{},"Tier",[544,548,549],{},"What it is",[544,551,552],{},"When to use it",[554,555,556,570,583,596],"tbody",{},[541,557,558,564,567],{},[559,560,561],"td",{},[464,562,563],{},"slim",[559,565,566],{},"A consistent slice of the catalogue",[559,568,569],{},"Day to day. Fast to pull, fast to boot.",[541,571,572,577,580],{},[559,573,574],{},[464,575,576],{},"partial",[559,578,579],{},"The full catalogue, sampled AI output",[559,581,582],{},"You need real breadth of vendors and products.",[541,584,585,590,593],{},[559,586,587],{},[464,588,589],{},"heavy",[559,591,592],{},"The current production export",[559,594,595],{},"Reproducing something that depends on data volume.",[541,597,598,603,606],{},[559,599,600],{},[464,601,602],{},"empty",[559,604,605],{},"Schema and migration history only",[559,607,608],{},"Onboarding, first-run behaviour, migrations.",[455,610,611,612,615,616,619,620,623],{},"Import ",[464,613,614],{},"esp-database-8"," and ",[464,617,618],{},"esp-postgres"," at the ",[459,621,622],{},"same tier",". They're used together, and mixing them gives search results pointing at content that isn't there.",[484,625,627],{"id":626},"refreshing-from-production","Refreshing from production",[455,629,630,631,634],{},"Do this when the shared data has gone stale. You'll need production access — see ",[469,632,217],{"href":633},"..\u002F..\u002Fdatabase-connection"," for the jump box setup.",[636,637,639],"h3",{"id":638},"_1-take-the-dumps","1. Take the dumps",[455,641,642],{},"Keep PuTTY open for this whole step; the tunnel dies with it.",[455,644,645],{},[459,646,647],{},"MySQL (Workbench)",[649,650,651,660,670,676,686],"ol",{},[652,653,654,655],"li",{},"Open MySQL Workbench and connect to the production database.\n",[656,657],"img",{"alt":658,"src":659},"MySQL Workbench Connection","\u002Fimages\u002Fengineering\u002Fdatabase-refresh\u002Fmysql-add-connection.png",[652,661,662,663,666,667,482],{},"Click ",[464,664,665],{},"Server"," > ",[464,668,669],{},"Data Export",[652,671,672,673,482],{},"Select the schema and tables, and choose ",[464,674,675],{},"Dump Structure and Data",[652,677,678,679,682,683,482],{},"Under ",[464,680,681],{},"Export Options",", choose ",[464,684,685],{},"Export to Self-Contained File",[652,687,662,688,482],{},[464,689,690],{},"Start Export",[455,692,693,694,697,698,701],{},"Export CPS twice — once as ",[464,695,696],{},"Dump Structure Only"," and once as ",[464,699,700],{},"Dump Data Only",". The data file is over 2 GB and a single file is awkward to re-run when something fails partway.",[455,703,704],{},[459,705,706],{},"PostgreSQL (PgAdmin)",[649,708,709,716,722,728,743],{},[652,710,711,712],{},"Open PgAdmin and connect to the production database.\n",[656,713],{"alt":714,"src":715},"PgAdmin Connection","\u002Fimages\u002Fengineering\u002Fdatabase-refresh\u002Fpostgres-add-connection.png",[652,717,718,719,482],{},"Right click the database and select ",[464,720,721],{},"Backup",[652,723,724,725,482],{},"Enter a filename, e.g. ",[464,726,727],{},"PGStagingDump20260901.sql",[652,729,730,731,462,734,615,737,462,740,482],{},"Set ",[464,732,733],{},"Format",[464,735,736],{},"Plain",[464,738,739],{},"File compression",[464,741,742],{},"None",[652,744,662,745,482],{},[464,746,721],{},[455,748,749,750,753],{},"Save everything into ",[464,751,752],{},"infra-devenv-pro\u002Fdumps\u002F",". The folder is gitignored, and the filenames don't matter — each file is identified by reading its header.",[755,756,757,760,761,764,765,767],"note",{},[464,758,759],{},"Custom"," format also works, but PgAdmin dumps with its own bundled ",[464,762,763],{},"pg_dump",", which is usually newer than the local container and produces an archive it can't read. The refresh works around that automatically; ",[464,766,736],{}," just avoids it.",[636,769,771],{"id":770},"_2-run-the-refresh","2. Run the refresh",[500,773,774],{"className":502,"code":503,"language":504,"meta":505,"style":505},[464,775,776],{"__ignoreMap":505},[509,777,778,780,782],{"class":511,"line":512},[509,779,482],{"class":515},[509,781,519],{"class":518},[509,783,523],{"class":522},[455,785,529,786,789],{},[464,787,788],{},"Database Refresh",". It finds the dumps, asks which tier you want, and does the rest.",[791,792,793,799,808],"ul",{},[652,794,795,798],{},[459,796,797],{},"CPS and platform"," are refreshed by default, including the Postgres vector store.",[652,800,801,804,805,482],{},[459,802,803],{},"CAS is offered separately"," and is off by default. Including it prunes the data to a single organisation and replaces every password — say yes and it asks what password you want, defaulting to ",[464,806,807],{},"ESProfiler@123",[652,809,810],{},"Other service schemas are listed and skipped.",[455,812,813,814,482],{},"Nothing is changed until you confirm, and every prompt has a ",[464,815,816],{},"Go Back",[455,818,819],{},"While it runs:",[535,821,822,832],{},[538,823,824],{},[541,825,826,829],{},[544,827,828],{},"Key",[544,830,831],{},"Effect",[554,833,834,844,854,864],{},[541,835,836,841],{},[559,837,838],{},[464,839,840],{},"d",[559,842,843],{},"Show debug output",[541,845,846,851],{},[559,847,848],{},[464,849,850],{},"i",[559,852,853],{},"Back to normal output",[541,855,856,861],{},[559,857,858],{},[464,859,860],{},"q",[559,862,863],{},"Stop cleanly after the current step",[541,865,866,871],{},[559,867,868],{},[464,869,870],{},"Ctrl-C",[559,872,873],{},"Stop now",[455,875,876,877,879],{},"If a run dies partway — a failed script, a closed laptop, Docker restarting — run ",[464,878,788],{}," again and it offers to resume from where it stopped. It won't re-import the 2 GB dump it already read.",[636,881,883],{"id":882},"_3-export-and-publish","3. Export and publish",[500,885,886],{"className":502,"code":503,"language":504,"meta":505,"style":505},[464,887,888],{"__ignoreMap":505},[509,889,890,892,894],{"class":511,"line":512},[509,891,482],{"class":515},[509,893,519],{"class":518},[509,895,523],{"class":522},[455,897,529,898,901],{},[464,899,900],{},"Export Volume",", then the volume and a tag matching the tier you built.",[535,903,904,914],{},[538,905,906],{},[541,907,908,911],{},[544,909,910],{},"Volume",[544,912,913],{},"Image",[554,915,916,927,938],{},[541,917,918,922],{},[559,919,920],{},[464,921,614],{},[559,923,924],{},[464,925,926],{},"ghcr.io\u002Fes-profiler\u002Fvolumes-esp-db-8:\u003Ctier>",[541,928,929,933],{},[559,930,931],{},[464,932,618],{},[559,934,935],{},[464,936,937],{},"ghcr.io\u002Fes-profiler\u002Fvolumes-esp-postgres:\u003Ctier>",[541,939,940,945],{},[559,941,942],{},[464,943,944],{},"esp-osnode-1",[559,946,947],{},[464,948,949],{},"ghcr.io\u002Fes-profiler\u002Fvolumes-esp-osdata1:\u003Ctier>",[455,951,952,953,615,956,959,960,963],{},"You'll need a GitHub token with ",[464,954,955],{},"write:packages",[464,957,958],{},"read:packages",", with SSO enabled for the ",[464,961,962],{},"es-profiler"," org. Dev Envi Pro walks you through creating one if the push fails.",[965,966,967,968,970],"warning",{},"Volume images are pushed to a container registry, so anything left in the data is published. Always let ",[464,969,788],{}," prepare the data — never export a volume you made by importing a dump straight in.",[484,972,974],{"id":973},"building-the-other-tiers","Building the other tiers",[455,976,977,978,980,981,984],{},"You don't need to import again for each tier. After a refresh, ",[464,979,788],{}," offers ",[464,982,983],{},"Reduce current data to a lower tier",", which produces exactly the same result as refreshing at that tier would.",[455,986,987],{},"So one import covers all four:",[500,989,994],{"className":990,"code":992,"language":993},[991],"language-text","refresh at heavy   ->  export heavy\nreduce to partial  ->  export partial\nreduce to slim     ->  export slim\nreduce to empty    ->  export empty\n","text",[464,995,992],{"__ignoreMap":505},[455,997,998],{},"Reduction only goes downwards. To get back to a bigger tier, import that volume or restore a backup.",[484,1000,1002],{"id":1001},"restoring-a-backup","Restoring a backup",[455,1004,1005,1006,1009,1010,1012,1013,482],{},"Before changing anything, the refresh offers to archive your current volumes into ",[464,1007,1008],{},"infra-devenv-pro\u002Fbackups\u002F",". To roll back, run ",[464,1011,788],{}," and choose ",[464,1014,1015],{},"Restore a pre-refresh backup",[455,1017,1018,1020,1021,1024],{},[464,1019,532],{}," can't do this — it pulls from the registry, and these backups are local files. Note a backup covers the ",[459,1022,1023],{},"whole volume",", so restoring reverts every schema.",[484,1026,203],{"id":1027},"troubleshooting",[455,1029,1030,1033,1034,1037,1038,1041,1042,1045],{},[459,1031,1032],{},"Migrations re-run on first boot and fail."," The Liquibase changelog was emptied — it has to survive every reduction. The table name differs by service: ",[464,1035,1036],{},"DB_CHANGELOG"," in platform, ",[464,1039,1040],{},"db_changelog"," in CAS and CPS, ",[464,1043,1044],{},"postgres_db_changelog"," in Postgres.",[455,1047,1048,1054],{},[459,1049,1050,1051,1053],{},"A schema still looks huge after an ",[464,1052,602],{}," refresh."," Probably stale statistics rather than stale data. Check the rows, not the reported size:",[500,1056,1058],{"className":502,"code":1057,"language":504,"meta":505,"style":505},"docker exec -i esp-database-8 mysql --user=root --password=ESProfiler@123 -e \"SELECT COUNT(*) FROM \\`esp-cps\\`.vendor;\"\n",[464,1059,1060],{"__ignoreMap":505},[509,1061,1062,1066,1069,1072,1075,1078,1081,1084,1087,1091,1094,1097,1100,1102,1105],{"class":511,"line":512},[509,1063,1065],{"class":1064},"sBMFI","docker",[509,1067,1068],{"class":522}," exec",[509,1070,1071],{"class":522}," -i",[509,1073,1074],{"class":522}," esp-database-8",[509,1076,1077],{"class":522}," mysql",[509,1079,1080],{"class":522}," --user=root",[509,1082,1083],{"class":522}," --password=ESProfiler@123",[509,1085,1086],{"class":522}," -e",[509,1088,1090],{"class":1089},"sMK4o"," \"",[509,1092,1093],{"class":522},"SELECT COUNT(*) FROM ",[509,1095,1096],{"class":518},"\\`",[509,1098,1099],{"class":522},"esp-cps",[509,1101,1096],{"class":518},[509,1103,1104],{"class":522},".vendor;",[509,1106,1107],{"class":1089},"\"\n",[455,1109,1110,1113,1114,1117,1118,1121],{},[459,1111,1112],{},"platform-api won't start after a Postgres refresh."," Production keeps its Postgres objects in an ",[464,1115,1116],{},"esp_platform"," schema, while the app's dev profile reads ",[464,1119,1120],{},"public",". The refresh moves everything across after restoring, so this should not happen — but if you imported by hand, you need to do the same:",[500,1123,1125],{"className":502,"code":1124,"language":504,"meta":505,"style":505},"docker exec -i esp-postgres psql --username=postgres --dbname=esp-platform -c \"ALTER EXTENSION \\\"uuid-ossp\\\" SET SCHEMA public; ALTER TABLE esp_platform.vector_store SET SCHEMA public;\"\n",[464,1126,1127],{"__ignoreMap":505},[509,1128,1129,1131,1133,1135,1138,1141,1144,1147,1150,1152,1155,1158,1161,1163,1166],{"class":511,"line":512},[509,1130,1065],{"class":1064},[509,1132,1068],{"class":522},[509,1134,1071],{"class":522},[509,1136,1137],{"class":522}," esp-postgres",[509,1139,1140],{"class":522}," psql",[509,1142,1143],{"class":522}," --username=postgres",[509,1145,1146],{"class":522}," --dbname=esp-platform",[509,1148,1149],{"class":522}," -c",[509,1151,1090],{"class":1089},[509,1153,1154],{"class":522},"ALTER EXTENSION ",[509,1156,1157],{"class":518},"\\\"",[509,1159,1160],{"class":522},"uuid-ossp",[509,1162,1157],{"class":518},[509,1164,1165],{"class":522}," SET SCHEMA public; ALTER TABLE esp_platform.vector_store SET SCHEMA public;",[509,1167,1107],{"class":1089},[455,1169,1170,1171,1174,1175,482],{},"Note ",[464,1172,1173],{},"CREATE EXTENSION IF NOT EXISTS"," is database-scoped, not schema-scoped, so the app's own migration can't fix this for you — it sees the extension already present somewhere and creates nothing in ",[464,1176,1120],{},[455,1178,1179,1184,1185,1187,1188,1190],{},[459,1180,1181,482],{},[464,1182,1183],{},"unsupported version (1.16) in file header"," The archive came from a newer ",[464,1186,763],{}," than the container has. The refresh retries with a matching Postgres image automatically; dumping as ",[464,1189,736],{}," avoids it.",[455,1192,1193,1194,1197,1198,1201],{},"Worth knowing: the local Postgres image is currently ",[459,1195,1196],{},"15",", while production runs ",[459,1199,1200],{},"16",", so every Postgres refresh is a version downgrade. It works for the vector store, but it's a mismatch worth fixing at the image level rather than working around.",[455,1203,1204,1209,1210,1213,1214,1217],{},[459,1205,1206,482],{},[464,1207,1208],{},"GTID_PURGED can only be set when GTID_EXECUTED is empty"," Re-export with ",[464,1211,1212],{},"--set-gtid-purged=OFF",", or delete the ",[464,1215,1216],{},"SET @@GLOBAL.GTID_PURGED=..."," line near the top of the dump.",[1219,1220],"hr",{},[484,1222,1224],{"id":1223},"reference","Reference",[636,1226,1228],{"id":1227},"how-a-refresh-works","How a refresh works",[455,1230,1231],{},"Each database goes through six steps:",[500,1233,1236],{"className":1234,"code":1235,"language":993},[991],"create temp  ->  import  ->  prepare  ->  verify  ->  swap  ->  cleanup\n",[464,1237,1235],{"__ignoreMap":505},[455,1239,1240,1241,1244,1245,1248],{},"Everything happens in a temp schema (",[464,1242,1243],{},"esp-cps__refresh","), and the live schema isn't touched until the swap. A failure before that leaves your environment working. Progress is recorded per step in ",[464,1246,1247],{},"dumps\u002F.refresh-state.json",", which is what makes resuming possible.",[455,1250,1251,1252,1255],{},"The ",[464,1253,1254],{},"verify"," step checks the result before it goes live: table counts, that every view resolves, that the migration changelog survived, and — on Postgres — that the application's own role can actually read it.",[636,1257,1259],{"id":1258},"what-each-tier-keeps","What each tier keeps",[455,1261,1262,1263,1265,1266,1268,1269,1268,1271,1273,1274,1276],{},"Tiers are cumulative. ",[464,1264,563],{}," is ",[464,1267,589],{},", then ",[464,1270,576],{},[464,1272,563],{},", so each script only describes what that step removes. ",[464,1275,602],{}," is the exception and stands alone.",[455,1278,1279,1283],{},[459,1280,1281],{},[464,1282,589],{}," removes what is meaningless outside the environment it came from, or must never be published: scheduler state, the Spring Batch job repository, audit events, Envers history, AI chat transcripts, uploaded files, the rebuildable search index, and personal contact details.",[455,1285,1286,1290],{},[459,1287,1288],{},[464,1289,576],{}," separates what the stack regenerates from what it only needs a sample of:",[791,1292,1293,1300,1306],{},[652,1294,1295,1299],{},[1296,1297,1298],"em",{},"Sampled"," — 5 capability collections with their mappings, the active frameworks at their latest version, and a capped slice of the framework decision cache.",[652,1301,1302,1305],{},[1296,1303,1304],{},"Kept"," — insights, central relations, notebooks and user tasks. They're small, and the screens that read them are useless when empty.",[652,1307,1308,1311],{},[1296,1309,1310],{},"Dropped"," — deep research, the threat feed, discovery findings, assertions, drafts, product similarity and company enrichment. All re-fetched or re-derived.",[455,1313,1314,1318,1319,1321],{},[459,1315,1316],{},[464,1317,563],{}," caps the catalogue itself. It picks a set of vendors, derives their products and features, and removes the rest, so nothing is left dangling. It also narrows what ",[464,1320,576],{}," sampled.",[455,1323,1324,1328],{},[459,1325,1326],{},[464,1327,602],{}," truncates every table except the migration changelog.",[636,1330,1332],{"id":1331},"cas-is-multi-tenant","CAS is multi-tenant",[455,1334,1335],{},"The CAS export contains every customer's users, groups, policies and SAML certificates. The refresh keeps one (ESPROFILER) and deletes the rest, then replaces every password, clears MFA secrets, and disables external SSO so local login works without an identity provider.",[455,1337,1338,1339,482],{},"This happens at ",[459,1340,1341,1342],{},"every tier, including ",[464,1343,602],{},[455,1345,1346],{},"To keep a different organisation, list them first:",[500,1348,1352],{"className":1349,"code":1350,"language":1351,"meta":505,"style":505},"language-sql shiki shiki-themes material-theme-lighter material-theme material-theme-palenight","SELECT BIN_TO_UUID(id), name, alias FROM organisations;\n","sql",[464,1353,1354],{"__ignoreMap":505},[509,1355,1356],{"class":511,"line":512},[509,1357,1350],{},[455,1359,1360,1361,1364,1365,482],{},"and set ",[464,1362,1363],{},"@ORG_UUID"," at the top of ",[464,1366,1367],{},"sql\u002Fcas\u002Fcas-00-prune-organisation.sql",[636,1369,1371],{"id":1370},"why-the-data-gets-rebuilt-at-the-end","Why the data gets rebuilt at the end",[455,1373,1374,1375,1378],{},"Deleting rows doesn't shrink a MySQL table. InnoDB — the storage engine MySQL uses — marks the rows dead and leaves them in the file. So a tier that prunes with ",[464,1376,1377],{},"DELETE"," frees almost no disk on its own, and the deleted rows stay readable inside the image you publish.",[455,1380,1381,1382,1385,1386,1389],{},"That's why ",[464,1383,1384],{},"prepare"," finishes by rebuilding every table from its live rows (",[464,1387,1388],{},"sql\u002Flib\u002Fmysql-99-reclaim.sql","). It's the single biggest factor in how large the published volume turns out, and the reason the tier scripts shouldn't be run without it.",[455,1391,1392,1393,1396],{},"While it runs, MySQL prints ",[464,1394,1395],{},"Table does not support optimize, doing recreate + analyze instead"," for each table — expected, not an error.",[636,1398,1400],{"id":1399},"running-the-sql-on-its-own","Running the SQL on its own",[455,1402,1403,1404,1407],{},"The scripts live in ",[464,1405,1406],{},"infra-devenv-pro\u002Fsql\u002F"," and work against a database you imported by hand:",[500,1409,1411],{"className":502,"code":1410,"language":504,"meta":505,"style":505},".venv\\Scripts\\python sql\\prepare.py --tier slim\n",[464,1412,1413],{"__ignoreMap":505},[509,1414,1415,1418,1421,1424,1427,1430],{"class":511,"line":512},[509,1416,1417],{"class":1064},".venv\\Scripts\\python",[509,1419,1420],{"class":522}," sql",[509,1422,1423],{"class":518},"\\p",[509,1425,1426],{"class":522},"repare.py",[509,1428,1429],{"class":522}," --tier",[509,1431,1432],{"class":522}," slim\n",[455,1434,1435,1436,1439,1440,1443,1444,1446,1447,1449],{},"Add ",[464,1437,1438],{},"--dry-run"," to see what it would run. Tier definitions live in ",[464,1441,1442],{},"sql\u002Ftiers.py"," and are shared with ",[464,1445,788],{},", so the two can't disagree about what ",[464,1448,563],{}," means.",[455,1451,1452,1453,1456,1457,1460],{},"Scrubbing CAS needs a password to hash, which needs the ",[464,1454,1455],{},"bcrypt"," package from ",[464,1458,1459],{},"requirements.txt",":",[500,1462,1464],{"className":502,"code":1463,"language":504,"meta":505,"style":505},".venv\\Scripts\\python sql\\prepare.py --tier slim --scope cas --cas-password \"ESProfiler@123\"\n",[464,1465,1466],{"__ignoreMap":505},[509,1467,1468,1470,1472,1474,1476,1478,1481,1484,1487,1490,1492,1494],{"class":511,"line":512},[509,1469,1417],{"class":1064},[509,1471,1420],{"class":522},[509,1473,1423],{"class":518},[509,1475,1426],{"class":522},[509,1477,1429],{"class":522},[509,1479,1480],{"class":522}," slim",[509,1482,1483],{"class":522}," --scope",[509,1485,1486],{"class":522}," cas",[509,1488,1489],{"class":522}," --cas-password",[509,1491,1090],{"class":1089},[509,1493,807],{"class":522},[509,1495,1107],{"class":1089},[636,1497,1499],{"id":1498},"safety-guards","Safety guards",[455,1501,1502,1503,1506,1507,1510,1511,1514],{},"Every script refuses to run unless ",[464,1504,1505],{},"@CONFIRM_LOCAL_COPY"," is set, which the drivers supply. The platform scripts additionally refuse to run anywhere except the local ",[464,1508,1509],{},"esp-platform"," schema — production platform data lives in ",[464,1512,1513],{},"esp-staging",", so a script accidentally pointed down the jump box tunnel stops before doing anything.",[455,1516,1517,1518,615,1520,1522],{},"Neither driver takes a host or credential argument. They only talk to the local ",[464,1519,614],{},[464,1521,618],{}," containers, so there's nothing to point at production.",[1524,1525,1526],"style",{},"html pre.shiki code .s2Zo4, html code.shiki .s2Zo4{--shiki-light:#6182B8;--shiki-default:#82AAFF;--shiki-dark:#82AAFF}html pre.shiki code .sTEyZ, html code.shiki .sTEyZ{--shiki-light:#90A4AE;--shiki-default:#EEFFFF;--shiki-dark:#BABED8}html pre.shiki code .sfazB, html code.shiki .sfazB{--shiki-light:#91B859;--shiki-default:#C3E88D;--shiki-dark:#C3E88D}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sBMFI, html code.shiki .sBMFI{--shiki-light:#E2931D;--shiki-default:#FFCB6B;--shiki-dark:#FFCB6B}html pre.shiki code .sMK4o, html code.shiki .sMK4o{--shiki-light:#39ADB5;--shiki-default:#89DDFF;--shiki-dark:#89DDFF}",{"title":505,"searchDepth":1528,"depth":1528,"links":1529},2,[1530,1531,1537,1538,1539,1540],{"id":486,"depth":1528,"text":487},{"id":626,"depth":1528,"text":627,"children":1532},[1533,1535,1536],{"id":638,"depth":1534,"text":639},3,{"id":770,"depth":1534,"text":771},{"id":882,"depth":1534,"text":883},{"id":973,"depth":1528,"text":974},{"id":1001,"depth":1528,"text":1002},{"id":1027,"depth":1528,"text":203},{"id":1223,"depth":1528,"text":1224,"children":1541},[1542,1543,1544,1545,1546,1547],{"id":1227,"depth":1534,"text":1228},{"id":1258,"depth":1534,"text":1259},{"id":1331,"depth":1534,"text":1332},{"id":1370,"depth":1534,"text":1371},{"id":1399,"depth":1534,"text":1400},{"id":1498,"depth":1534,"text":1499},"Pulling a shared database volume for local development, and refreshing it from production when the data gets stale.","md",null,{},true,{"title":152,"description":1548},"gnCR598XV8-FWFvZ7Rt4oyu_2kkI2LcqCb3qQH4FdIE",[1556,1558],{"title":143,"path":144,"stem":145,"description":1557,"children":-1},"Step-by-step guide for backend developers on adding new prompts, modifying existing prompts across platform-api and api-cps, integrating with Langfuse and esp-prompts, Quartz reload scheduling, and coordinating Data\u002FAI team benchmarking.",{"title":161,"path":162,"stem":163,"description":1559,"children":-1},"How we tag platform elements with data- attributes for Cypress E2E tests and PostHog reporting.",1795106631943]