Open fardarter opened 1 year ago
garethbowen
commented
1 year ago @fardarter Nice feature. I think a more sustainable approach would be to mount a volume which contains couchdb configuration overrides. If we do it right, then couch will load the default config, overridden by the CHT default config, overridden by any additional config you add. The benefit of this approach is it'll continue to work after upgrading. Would that solution suit?
fardarter
commented
1 year ago It's a viable approach. What I value about crudini is that I can modify only subsections of the ini file or achieve a composite of the three. Volumes probably make more sense with secrets involved anyway (I'm using sops to decrypt on our VM so secret isn't even in env vars for us).
Re other aspects of our solution (ideally would like to produce git diffs from this), this is my build script (mjs file):
import * as esbuild from 'esbuild'
import { copy } from 'esbuild-plugin-copy';
import { execSync } from 'child_process';
console.time('Build time');
const isString = value => typeof value === 'string' || value instanceof String;
const watchIsString = isString(process.env?.BUILD_WATCH)
const shouldWatch = watchIsString ? process.env?.BUILD_WATCH.toLowerCase() === 'true' : false
// See command references here:
// - https://stackoverflow.com/a/63438492 (from https://stackoverflow.com/questions/13713101/rsync-exclude-according-to-gitignore-hgignore-svnignore-like-filter-c)
// - https://www.redhat.com/sysadmin/sync-rsync
execSync("rsync -rlptgo --include='/.gitignore' --exclude='/.git' --filter=':- .gitignore' --delete-after ./src/cht-core-source/ ./src/cht-core-build", (error, stdout, stderr) => {
if (error) {
console.log(`error: ${error.message}`);
return;
}
if (stderr) {
console.log(`stderr: ${stderr}`);
return;
}
console.log(`stdout: ${stdout}`);
});
// https://www.redhat.com/sysadmin/sync-rsync
execSync("rsync -rlptgo --include='/.gitignore' --exclude='/.git' --filter=':- .gitignore' ./src/cht-core-patch/ ./src/cht-core-build", (error, stdout, stderr) => {
if (error) {
console.log(`error: ${error.message}`);
return;
}
if (stderr) {
console.log(`stderr: ${stderr}`);
return;
}
console.log(`stdout: ${stdout}`);
});
if(shouldWatch) {
const context = await esbuild.context({
plugins: [
copy({
resolveFrom: 'cwd',
once: true,
assets: {
from: ['src/cht-core-patch/**/*'],
to: ['src/cht-core-build'],
watch: shouldWatch,
},
})
],
})
await context.rebuild()
console.timeEnd('Build time');
await context.watch()
console.log("Watch mode enabled")
context.dispose()
} else {
console.timeEnd('Build time');
}Example of the above is that I have a patch folder, for example, with a gruntfile extension, and it's in the file tree for correct access scope.
Below is how I've extended the gruntfile (amend env outputs the docker tags to the .env file on a flag -- not written dedupe yet but it's trivial, I think. Have broken out the image build from tests for local use and also pipeline ordering). This would live at /cht/src/cht-core-patch/scripts/build/grunt-tasks/extension.js.
const buildVersions = require('../versions');
const { USE_PINNED_DOCKER_TAGS = false } = process.env
const fs = require('fs');
const path = require('path');
const getConfigDirs = () => {
return fs
.readdirSync('config')
.filter(file => fs.lstatSync(`config/${file}`).isDirectory());
};
const isString = value => typeof value === 'string' || value instanceof String;
const isTrue = value => isString(value) ? value.toLowerCase() === 'true' : value === true
const rootContext = path.resolve(__dirname, '../../../../../');
const targetFile = `${rootContext}/.env`
const env = fs.readFileSync(`${targetFile}`, 'utf8')
const envString = `${env}`
module.exports = function (grunt) {
let conf = {
exec: {
'phdc-amend-env': {
cmd: () => {
modifyEnv();
return 'echo "Finished amending .env file.'
},
stdio: 'inherit', // enable colors!
},
'phdc-setup-config-folders': {
cmd: () => {
const dirs = getConfigDirs();
const installDirs = dirs.flatMap(dir => {
return [
`cd config/${dir}`,
'npm ci',
"cd ../"
]
}).join(' && ')
return 'echo did run'
},
stdio: 'inherit', // enable colors!
},
'phdc-tag-images': {
cmd: () => {
if(buildVersions.ADDITIONAL_TAGS.length === 0) {
return 'echo "No additional tags"'
}
return [...buildVersions.SERVICES, ...buildVersions.INFRASTRUCTURE].flatMap(service => {
return buildVersions.ADDITIONAL_TAGS.flatMap(tag => {
return [`docker tag ${buildVersions.getImageTag(service)} ${buildVersions.getImageBase(service)}:${tag}`]
}).join(' && ');
}
).join(' && ');
},
stdio: 'inherit', // enable colors!
},
'phdc-push-images': {
cmd: () => {
return [...buildVersions.SERVICES, ...buildVersions.INFRASTRUCTURE].flatMap(service => {
return [`docker push ${buildVersions.getImageTag(service)}`, buildVersions.ADDITIONAL_TAGS.flatMap(tag => {
return [`docker push ${buildVersions.getImageBase(service)}:${tag}`]
}).join(' && ')];
}
).join(' && ');
},
stdio: 'inherit', // enable colors!
}
}
}
grunt.config.merge(conf);
grunt.registerTask('phdc-setup', 'Prepare all static, generated and installed files', [
'exec:phdc-amend-env',
'exec:phdc-setup-config-folders',
'install-dependencies',
'build',
"copy-static-files-to-api",
'uglify:api',
'cssmin:api',
]);
grunt.registerTask('phdc-setup-dev', 'Prepare all static, generated and installed files', [
'exec:phdc-amend-env',
'exec:phdc-setup-config-folders',
'install-dependencies',
'build-dev',
]);
const hasInfrastructure = buildVersions.INFRASTRUCTURE.length > 0
// Which images are built is governed by the scripts/build/versions.js file.
grunt.registerTask('phdc-scratch-build-images', '', hasInfrastructure ? [
'phdc-setup',
'exec:build-service-images',
'exec:build-images',
'exec:phdc-tag-images'
] : [
'phdc-setup',
'exec:build-service-images',
'exec:phdc-tag-images'
]);
grunt.registerTask('phdc-scratch-build-and-push-images', '', [
'phdc-scratch-build-images',
'exec:phdc-push-images'
]);
};
function modifyEnv() {
const lines = envString.split("\n");
let existingMap = new Map();
lines.forEach(line => {
const [first, second = ""] = line.split("=")
if (first != "") {
existingMap.set(first, second);
}
})
let filestring = `${envString.trimEnd()}\n`;
if (isTrue(USE_PINNED_DOCKER_TAGS) === false) {
[...buildVersions.SERVICES, ...buildVersions.INFRASTRUCTURE].forEach(service => {
const key = `CHT_${service}_DOCKER_IMAGE`.replace("-", "_").toUpperCase()
const value = buildVersions.getImageTag(service)
if (existingMap.has(key) === true && existingMap.get(key) !== `${value}`) {
const existingval = `${key}=${existingMap.get(key)}\n`
existingMap.set(key, `${value}`)
filestring = filestring.replace(existingval, `${key}=${value}\n`)
}
if (existingMap.has(key) === false) {
existingMap.set(key, `${value}`)
filestring = filestring.concat(`${key}=${value}\n`)
}
})
}
if (isTrue(USE_PINNED_DOCKER_TAGS) === true) {
let keySet = new Set();
[...buildVersions.SERVICES, ...buildVersions.INFRASTRUCTURE].forEach(service => {
const key = `CHT_${service}_DOCKER_IMAGE`.replace("-", "_").toUpperCase()
keySet.add(key)
})
const filteredLines = lines.filter(item => {
const trimmed = item.trim()
if (trimmed.charAt(0) === "#") {
return true;
}
if (keySet.has(item.split("=")[0])) {
return false
}
return true
})
filestring = filteredLines.join("\n");
}
console.info(`Writing to: ${targetFile}`)
fs.writeFileSync(targetFile, filestring, { flag: 'w+' }, err => {
if (err) {
console.error(err);
}
});
}
What feature do you want to improve? I want to make the CHT Core repo extensible so that downstream consumers have a structured way of building their own versions while keeping patches and changes distinct enough that they're easy to understand and potentially contribute upstream.
We're building for Azure so need (will need) to patch in things like application insights and blob storage connections, as well as B2C for authorisation.
Describe the improvement you'd like I have a somewhat working solution which merges a source (git module) repo with a patch folder into a build folder, which is what we actually test against/deploy. I've set up a meta package.json for all the scripting and have extended the gruntfile the
--tasksflag.Honestly would just like a chance to show someone how it all works? I'd like to do less work anyway, and a userspace config zone would help.
Describe alternatives you've considered I mean I've built an approach. It could do with work, but it's got a lot going for it.
Additional context So, just a methodology that made @dianabarsan ask me to raise an issue, though honestly not sure how to present it. It's just a technique for altering ini files with another file (echo could echo a file).
Having a default and a stipulated location a script might look for a merge file would be one way of giving userspace control.