HorizonCode/SysBackup
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

move backup files/dirs into temp dir to prevent size mismatch error

Browse Source
This commit is contained in:
HorizonCode 2022-08-09 09:29:43 +02:00
parent 62a8da129a
commit 00b886740d
1 changed files with 159 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

216
index.js
View File

@ -8,7 +8,9 @@ const ora = require('ora');
const BackupArchive = require('./archiver'); const BackupArchive = require('./archiver');
const mysqldump = require('mysqldump'); const mysqldump = require('mysqldump');
const CURRENT_FOLDER = __dirname.startsWith('/snapshot') ? path.dirname(process.env._) : __dirname; const CURRENT_FOLDER = __dirname.startsWith('/snapshot') ?
path.dirname(process.env._) :
__dirname;
const TASKS_DIRECTORY = path.join(CURRENT_FOLDER, 'tasks'); const TASKS_DIRECTORY = path.join(CURRENT_FOLDER, 'tasks');
const BACKUPS_DIRECTORY = path.join(CURRENT_FOLDER, 'backups'); const BACKUPS_DIRECTORY = path.join(CURRENT_FOLDER, 'backups');
const TEMP_DIR = path.join(CURRENT_FOLDER, '.temp'); const TEMP_DIR = path.join(CURRENT_FOLDER, '.temp');
@ -19,7 +21,8 @@ const run = async (programArgs) => {
case 'exit': case 'exit':
logger.info('exiting...'); logger.info('exiting...');
break; break;
case 'createcrontab': { case 'createcrontab':
{
logger.info('not added yet.'); logger.info('not added yet.');
} }
break; break;
@ -44,7 +47,10 @@ const run = async (programArgs) => {
name: 'filename', name: 'filename',
message: 'Select a Task file to use', message: 'Select a Task file to use',
choices: AUTO_COMPLETE_ARRAY, choices: AUTO_COMPLETE_ARRAY,
validate: (value) => FILES.includes(value + '.json') ? true : 'Task file does not exist, please specify a valid task file.', validate: (value) =>
FILES.includes(value + '.json') ?
true :
'Task file does not exist, please specify a valid task file.',
}); });
taskName = PROMPT.filename; taskName = PROMPT.filename;
@ -59,25 +65,57 @@ const run = async (programArgs) => {
process.exit(1); process.exit(1);
} }
const TASK_FILE_CONTENTS = JSON.parse(fs.readFileSync(path.join(TASKS_DIRECTORY, taskName), 'utf8')); const TASK_FILE_CONTENTS = JSON.parse(
fs.readFileSync(path.join(TASKS_DIRECTORY, taskName), 'utf8'),
);
const VALID_TASK_FILE = validate(TASK_FILE_CONTENTS); const VALID_TASK_FILE = validate(TASK_FILE_CONTENTS);
if (VALID_TASK_FILE.length > 0) { if (VALID_TASK_FILE.length > 0) {
logger.error('task file is not valid, ' + VALID_TASK_FILE.length + ' errors found'); logger.error(
logger.info('check with \'' + process.argv0 + ' checkTaskConf ' + path.parse(taskName).name + '\''); 'task file is not valid, ' + VALID_TASK_FILE.length + ' errors found',
);
logger.info(
'check with \'' +
process.argv0 +
' checkTaskConf ' +
path.parse(taskName).name +
'\'',
);
process.exit(1); process.exit(1);
} }
logger.success('task file is valid'); logger.success('task file is valid');
const TIMER = new (require('./timer'))().startTimer(); const TIMER = new (require('./timer'))().startTimer();
const SPINNER = ora('initialization...').start(); const SPINNER = ora('preparing backup... please wait.').start();
if (!fs.existsSync(path.join(BACKUPS_DIRECTORY, path.parse(taskName).name))) {
await fs.promises.mkdir(path.join(BACKUPS_DIRECTORY, path.parse(taskName).name)); const FILE_PATHS = [];
const fsExtra = require('fs-extra');
await Promise.all(TASK_FILE_CONTENTS.filesystem.targets.map(async (file) => {
if (fs.existsSync(file)) {
const newPath = path.join(TEMP_DIR, path.parse(file).name);
FILE_PATHS.push(newPath);
await fsExtra.copySync(file, newPath);
}
}));
if (
!fs.existsSync(path.join(BACKUPS_DIRECTORY, path.parse(taskName).name))
) {
await fs.promises.mkdir(
path.join(BACKUPS_DIRECTORY, path.parse(taskName).name),
);
} else { } else {
if (TASK_FILE_CONTENTS.vacuum.enabled) { if (TASK_FILE_CONTENTS.vacuum.enabled) {
SPINNER.text = 'cleaning up old backups...'; SPINNER.text = 'cleaning up old backups...';
const ALL_FILES = fs.readdirSync(path.join(BACKUPS_DIRECTORY, path.parse(taskName).name)); const ALL_FILES = fs.readdirSync(
path.join(BACKUPS_DIRECTORY, path.parse(taskName).name),
);
const CURRENT_DATE = Date.now(); const CURRENT_DATE = Date.now();
for (const FILE of ALL_FILES) { for (const FILE of ALL_FILES) {
const FILE_DATE = new Date(fs.statSync(path.join(BACKUPS_DIRECTORY, path.parse(taskName).name, FILE)).birthtime).getTime(); const FILE_DATE = new Date(
fs.statSync(
path.join(BACKUPS_DIRECTORY, path.parse(taskName).name, FILE),
).birthtime,
).getTime();
let timeAdd = 0; let timeAdd = 0;
switch (TASK_FILE_CONTENTS.vacuum.unit) { switch (TASK_FILE_CONTENTS.vacuum.unit) {
case 'DAYS': case 'DAYS':
@ -92,22 +130,34 @@ const run = async (programArgs) => {
} }
const DELETE_DATE = new Date(FILE_DATE + timeAdd); const DELETE_DATE = new Date(FILE_DATE + timeAdd);
if (DELETE_DATE.getTime() < CURRENT_DATE) { if (DELETE_DATE.getTime() < CURRENT_DATE) {
await fs.unlinkSync(path.join(BACKUPS_DIRECTORY, path.parse(taskName).name, FILE)); await fs.unlinkSync(
path.join(BACKUPS_DIRECTORY, path.parse(taskName).name, FILE),
);
} }
} }
} }
} }
const UNREPLACED_FILENAME = TASK_FILE_CONTENTS.general.outputFile; const UNREPLACED_FILENAME = TASK_FILE_CONTENTS.general.outputFile;
const REPLACED_FILENAME = UNREPLACED_FILENAME.replace('{date}', require('moment')().format(TASK_FILE_CONTENTS.general.dateFormat)) const REPLACED_FILENAME = UNREPLACED_FILENAME.replace(
.replace('{taskName}', path.parse(taskName).name); '{date}',
require('moment')().format(TASK_FILE_CONTENTS.general.dateFormat),
).replace('{taskName}', path.parse(taskName).name);
const DB_FILE = path.join(TEMP_DIR, require('crypto').randomBytes(16).toString('hex') + '.sql'); const DB_FILE = path.join(
const ARCHIVE = new BackupArchive(path.join(BACKUPS_DIRECTORY, path.parse(taskName).name, TEMP_DIR,
REPLACED_FILENAME + '.tar.gz'), require('crypto').randomBytes(16).toString('hex') + '.sql',
TASK_FILE_CONTENTS.filesystem.targets, );
const ARCHIVE = new BackupArchive(
path.join(
BACKUPS_DIRECTORY,
path.parse(taskName).name,
REPLACED_FILENAME + '.tar.gz',
),
FILE_PATHS,
TASK_FILE_CONTENTS.general.gzip, TASK_FILE_CONTENTS.general.gzip,
TASK_FILE_CONTENTS.general.gzipLevel); TASK_FILE_CONTENTS.general.gzipLevel,
);
ARCHIVE.eventEmitter.on('progress', (progressInfo) => { ARCHIVE.eventEmitter.on('progress', (progressInfo) => {
const TOTAL = ARCHIVE.totalFiles; const TOTAL = ARCHIVE.totalFiles;
@ -117,10 +167,12 @@ const run = async (programArgs) => {
}); });
ARCHIVE.eventEmitter.on('finish', async () => { ARCHIVE.eventEmitter.on('finish', async () => {
if (TASK_FILE_CONTENTS.mysql.enabled) { SPINNER.text = 'Cleaning up temporary files...';
if (DB_FILE !== undefined && fs.existsSync(DB_FILE)) { if (fs.existsSync(DB_FILE)) {
await fs.unlinkSync(DB_FILE); await fs.unlinkSync(DB_FILE);
} }
for (const FILE of FILE_PATHS) {
await fsExtra.removeSync(FILE);
} }
SPINNER.succeed('backup complete, took ' + TIMER.endTimer()); SPINNER.succeed('backup complete, took ' + TIMER.endTimer());
}); });
@ -178,7 +230,10 @@ const run = async (programArgs) => {
name: 'filename', name: 'filename',
message: 'Select a Task file to check', message: 'Select a Task file to check',
choices: AUTO_COMPLETE_ARRAY, choices: AUTO_COMPLETE_ARRAY,
validate: (value) => FILES.includes(value + '.json') ? true : 'Task file does not exist, please specify a valid task file.', validate: (value) =>
FILES.includes(value + '.json') ?
true :
'Task file does not exist, please specify a valid task file.',
}); });
taskName = PROMPT.filename; taskName = PROMPT.filename;
@ -198,7 +253,11 @@ const run = async (programArgs) => {
const VALIDATION_ERRORS = validate(TASK_FILE_TO_CHECK); const VALIDATION_ERRORS = validate(TASK_FILE_TO_CHECK);
if (VALIDATION_ERRORS.length > 0) { if (VALIDATION_ERRORS.length > 0) {
logger.error('task file is not valid, ' + VALIDATION_ERRORS.length + ' errors found'); logger.error(
'task file is not valid, ' +
VALIDATION_ERRORS.length +
' errors found',
);
for (const ERROR of VALIDATION_ERRORS) logger.error(ERROR); for (const ERROR of VALIDATION_ERRORS) logger.error(ERROR);
} else { } else {
logger.success('Task file is valid!'); logger.success('Task file is valid!');
@ -214,7 +273,10 @@ const run = async (programArgs) => {
type: 'text', type: 'text',
name: 'filename', name: 'filename',
message: 'Define a name for the new task file', message: 'Define a name for the new task file',
validate: (value) => value.length <= 0 || value == '' ? 'Please specify a valid file name' : true, validate: (value) =>
value.length <= 0 || value == '' ?
'Please specify a valid file name' :
true,
}); });
taskName = PROMPT.filename; taskName = PROMPT.filename;
@ -224,7 +286,8 @@ const run = async (programArgs) => {
const PROMPT_2 = await prompts({ const PROMPT_2 = await prompts({
type: 'toggle', type: 'toggle',
name: 'overwrite', name: 'overwrite',
message: 'A task file with the same name already exists. Do you want to overwrite it?', message:
'A task file with the same name already exists. Do you want to overwrite it?',
active: 'yes', active: 'yes',
inactive: 'no', inactive: 'no',
initial: false, initial: false,
@ -244,38 +307,37 @@ const run = async (programArgs) => {
const TASK_FILE_PATH = path.join(TASKS_DIRECTORY, taskName + '.json'); const TASK_FILE_PATH = path.join(TASKS_DIRECTORY, taskName + '.json');
const TASK_CONFIG = { const TASK_CONFIG = {
'general': { general: {
'dateFormat': 'yyyy-MM-DD HH-mm-ss', dateFormat: 'yyyy-MM-DD HH-mm-ss',
'outputFile': '{date} - {taskName}', outputFile: '{date} - {taskName}',
'gzip': true, gzip: true,
'gzipLevel': 6, gzipLevel: 6,
}, },
'vacuum': { vacuum: {
'enabled': true, enabled: true,
'unit': 'DAYS', unit: 'DAYS',
'time': 7, time: 7,
}, },
'mysql': { mysql: {
'enabled': true, enabled: true,
'host': 'localhost', host: 'localhost',
'port': 3306, port: 3306,
'user': '', user: '',
'password': '', password: '',
'database': '', database: '',
}, },
'filesystem': { filesystem: {
'enabled': true, enabled: true,
'targets': [ targets: ['/home/magento/', '/home/test/testfile.txt'],
'/home/magento/',
'/home/test/testfile.txt',
],
}, },
}; };
try { try {
const SAVE_FILE = fs.createWriteStream(TASK_FILE_PATH); const SAVE_FILE = fs.createWriteStream(TASK_FILE_PATH);
SAVE_FILE.write(JSON.stringify(TASK_CONFIG, null, 4)); SAVE_FILE.write(JSON.stringify(TASK_CONFIG, null, 4));
SAVE_FILE.end(); SAVE_FILE.end();
logger.success('Task file "' + path.basename(TASK_FILE_PATH) + '" saved successfully'); logger.success(
'Task file "' + path.basename(TASK_FILE_PATH) + '" saved successfully',
);
} catch (err) { } catch (err) {
logger.error(err); logger.error(err);
} }
@ -292,27 +354,39 @@ const validate = (taskConfig) => {
const ERRORS = []; const ERRORS = [];
if (taskConfig.general && typeof taskConfig.general === 'object') { if (taskConfig.general && typeof taskConfig.general === 'object') {
if (taskConfig.general.dateFormat && typeof taskConfig.general.dateFormat === 'string') { if (
taskConfig.general.dateFormat &&
typeof taskConfig.general.dateFormat === 'string'
) {
const DATE_FORMAT = taskConfig.general.dateFormat; const DATE_FORMAT = taskConfig.general.dateFormat;
const DATE_FORMAT_REGEX = /d{1,4}|D{3,4}|m{1,4}|yy(?:yy)?|([HhMsTt])\1?|W{1,2}|[LlopSZN]|"[^"]*"|'[^']*'/g; const DATE_FORMAT_REGEX =
/d{1,4}|D{3,4}|m{1,4}|yy(?:yy)?|([HhMsTt])\1?|W{1,2}|[LlopSZN]|"[^"]*"|'[^']*'/g;
if (!DATE_FORMAT_REGEX.test(DATE_FORMAT)) { if (!DATE_FORMAT_REGEX.test(DATE_FORMAT)) {
ERRORS.push('general.dateFormat is not a valid date format'); ERRORS.push('general.dateFormat is not a valid date format');
} }
} else { } else {
ERRORS.push('general.dateFormat is not defined or is not a string'); ERRORS.push('general.dateFormat is not defined or is not a string');
} }
if (taskConfig.general.outputFile && typeof taskConfig.general.outputFile === 'string') { if (
taskConfig.general.outputFile &&
typeof taskConfig.general.outputFile === 'string'
) {
const OUTPUT_FILE = taskConfig.general.outputFile; const OUTPUT_FILE = taskConfig.general.outputFile;
const OUTPUT_FILE_REGEX = /^([^.]+)$/g; const OUTPUT_FILE_REGEX = /^([^.]+)$/g;
if (!OUTPUT_FILE_REGEX.test(OUTPUT_FILE)) { if (!OUTPUT_FILE_REGEX.test(OUTPUT_FILE)) {
ERRORS.push('general.outputFile is not a valid output file name, maybe you added a file extension?'); ERRORS.push(
'general.outputFile is not a valid output file name, maybe you added a file extension?',
);
} }
} else { } else {
ERRORS.push('general.outputFile is not defined or is not a string'); ERRORS.push('general.outputFile is not defined or is not a string');
} }
if (typeof taskConfig.general.gzip === 'boolean') { if (typeof taskConfig.general.gzip === 'boolean') {
if (taskConfig.general.gzip) { if (taskConfig.general.gzip) {
if (!taskConfig.general.gzipLevel && typeof taskConfig.general.gzipLevel !== 'number') { if (
!taskConfig.general.gzipLevel &&
typeof taskConfig.general.gzipLevel !== 'number'
) {
ERRORS.push('general.gzipLevel is not defined or is not a number'); ERRORS.push('general.gzipLevel is not defined or is not a number');
} }
} }
@ -330,19 +404,29 @@ const validate = (taskConfig) => {
if (typeof taskConfig.vacuum.enabled !== 'boolean') { if (typeof taskConfig.vacuum.enabled !== 'boolean') {
ERRORS.push('vacuum.enabled is not defined or is not a boolean'); ERRORS.push('vacuum.enabled is not defined or is not a boolean');
} else if (taskConfig.vacuum.enabled) { } else if (taskConfig.vacuum.enabled) {
if (taskConfig.vacuum.unit && typeof taskConfig.vacuum.unit === 'string') { if (
taskConfig.vacuum.unit &&
typeof taskConfig.vacuum.unit === 'string'
) {
const UNIT = taskConfig.vacuum.unit; const UNIT = taskConfig.vacuum.unit;
const UNIT_REGEX = /^(DAYS|HOURS|MINUTES)$/g; const UNIT_REGEX = /^(DAYS|HOURS|MINUTES)$/g;
if (!UNIT_REGEX.test(UNIT)) { if (!UNIT_REGEX.test(UNIT)) {
ERRORS.push('vacuum.unit is not a valid unit, please use DAYS, HOURS or MINUTES'); ERRORS.push(
'vacuum.unit is not a valid unit, please use DAYS, HOURS or MINUTES',
);
} }
} else { } else {
ERRORS.push('vacuum.unit is not defined or is not a string'); ERRORS.push('vacuum.unit is not defined or is not a string');
} }
if (taskConfig.vacuum.time && typeof taskConfig.vacuum.time === 'number') { if (
taskConfig.vacuum.time &&
typeof taskConfig.vacuum.time === 'number'
) {
const TIME = taskConfig.vacuum.time; const TIME = taskConfig.vacuum.time;
if (TIME < 1) { if (TIME < 1) {
ERRORS.push('vacuum.time is not a valid time, please use a number greater than 0'); ERRORS.push(
'vacuum.time is not a valid time, please use a number greater than 0',
);
} }
} else { } else {
ERRORS.push('vacuum.time is not defined or is not a number'); ERRORS.push('vacuum.time is not defined or is not a number');
@ -371,7 +455,10 @@ const validate = (taskConfig) => {
if (typeof taskConfig.mysql.password !== 'string') { if (typeof taskConfig.mysql.password !== 'string') {
ERRORS.push('mysql.password is not defined or is not a string'); ERRORS.push('mysql.password is not defined or is not a string');
} }
if (!taskConfig.mysql.database || typeof taskConfig.mysql.database !== 'string') { if (
!taskConfig.mysql.database ||
typeof taskConfig.mysql.database !== 'string'
) {
ERRORS.push('mysql.database is not defined or is not a string'); ERRORS.push('mysql.database is not defined or is not a string');
} }
} }
@ -388,14 +475,19 @@ const validate = (taskConfig) => {
if (typeof taskConfig.filesystem.enabled !== 'boolean') { if (typeof taskConfig.filesystem.enabled !== 'boolean') {
ERRORS.push('filesystem.enabled is not defined or is not a boolean'); ERRORS.push('filesystem.enabled is not defined or is not a boolean');
} else if (taskConfig.filesystem.enabled) { } else if (taskConfig.filesystem.enabled) {
if (taskConfig.filesystem.targets && typeof taskConfig.filesystem.targets === 'object') { if (
taskConfig.filesystem.targets &&
typeof taskConfig.filesystem.targets === 'object'
) {
const TARGETS = taskConfig.filesystem.targets; const TARGETS = taskConfig.filesystem.targets;
if (TARGETS.length < 1) { if (TARGETS.length < 1) {
ERRORS.push('filesystem.targets is not defined or is not an array'); ERRORS.push('filesystem.targets is not defined or is not an array');
} else { } else {
for (const target of TARGETS) { for (const target of TARGETS) {
if (!target || typeof target !== 'string') { if (!target || typeof target !== 'string') {
ERRORS.push('filesystem.targets[] is not defined or is not a valid path'); ERRORS.push(
'filesystem.targets[] is not defined or is not a valid path',
);
} }
} }
} }
@ -434,7 +526,8 @@ const cli = async (forcePrompt) => {
const PROMPT = await prompts({ const PROMPT = await prompts({
type: 'toggle', type: 'toggle',
name: 'createTask', name: 'createTask',
message: 'This is the first time you run this script, do you want to create a new task?', message:
'This is the first time you run this script, do you want to create a new task?',
active: 'yes', active: 'yes',
inactive: 'no', inactive: 'no',
initial: false, initial: false,
@ -474,4 +567,3 @@ const cli = async (forcePrompt) => {
}; };
cli(false); cli(false);