<?php // Client for continuous integration and deployment // It can run on the same machine as the web server. error_reporting(E_ALL); ini_set('track_errors', 'On'); set_time_limit(0); define('RG_JOB_INIT', 1); define('RG_JOB_HELPER_STARTED', 2); define('RG_JOB_STARTED', 3); define('RG_JOB_ERROR', 4); define('RG_JOB_DONE', 10); $_s = microtime(TRUE); $INC = dirname(__FILE__) . "/../inc"; require_once($INC . "/init.inc.php"); require_once($INC . "/log.inc.php"); require_once($INC . "/prof.inc.php"); require_once($INC . "/builder.inc.php"); require_once($INC . "/conn.inc.php"); rg_prof_start('MAIN'); if (!isset($_SERVER['argv'][1])) $name = 'main'; else $name = $_SERVER['argv'][1]; if (!isset($_SERVER['argv'][2])) $conf_file = '/etc/rocketgit/worker.conf'; else $conf_file = $_SERVER['argv'][2]; // TODO: use different files for different workers! rg_log_set_file($rg_log_dir . '/worker-' . $name . '.log'); rg_log_set_sid("000000"); // to spread the logs rg_log('name=' . $name . ' conf_file=' . $conf_file); /* * Load configuration file */ function reload_config() { global $conf_file; global $conf; $_conf = @file($conf_file); if ($_conf === FALSE) { // worker.conf not found exit(0); } $last_key = FALSE; $conf = array('env' => array()); foreach ($_conf as $line) { $tline = trim($line); if (empty($tline)) continue; if (strncmp($tline, '#', 1) == 0) continue; $t = explode('=', $line, 2); if (count($t) != 2) { rg_log('Invalid line [' . $line . ']!'); continue; } $var = trim($t[0]); $value = trim($t[1]); if (strcmp($var, 'env') == 0) { $conf['env'][$value] = array('paras' => ''); $last_parent = &$conf['env'][$value]; } else if ((strncmp($line, " ", 1) == 0) || (strncmp($line, "\t", 1) == 0)) { if ($last_parent === FALSE) { rg_log('Invalid line [' . $line . ']!'); continue; } $t = explode('=', $line, 2); if (count($t) != 2) { rg_log('Invalid line [' . $line . ']!'); continue; } $var = trim($t[0]); $value = trim($t[1]); $last_parent[$var] = $value; } else { $conf[$var] = $value; $last_parent = FALSE; } } if (!file_exists($conf['state'] . '/key.pub')) { rg_log('Creating SSH key...'); $cmd = 'ssh-keygen -t rsa -b 4096 -N \'\'' . ' -C \'Key to connect to builder\'' . ' -f ' . escapeshellarg($conf['state'] . '/key'); $r = rg_exec($cmd, '', FALSE, FALSE); if ($r['ok'] != 1) { rg_log('Cannot create key: ' . $r['errmsg'] . '!'); sleep(60); exit(0); } } $conf['ssh_key'] = @file_get_contents($conf['state'] . '/key.pub'); if ($conf['ssh_key'] === FALSE) { rg_log('Cannot load key!'); sleep(60); exit(0); } rg_log_ml('conf: ' . print_r($conf, TRUE)); } /* * Starts an worker */ function start_worker($job) { global $conf; global $php_errormsg; $env = $conf['env'][$job['env']]; //rg_log_ml('DEBUG: env: ' . print_r($env, TRUE)); $jid = $job['id']; $emain = escapeshellarg($job['main']); $name = 'rg-worker-' . $job['id']; $ename = escapeshellarg($name); $master = escapeshellarg($env['image']); $img = escapeshellarg($job['main'] . '/image.qcow2'); $img2 = escapeshellarg($job['main'] . '/image2.raw'); $do_umount = FALSE; $err = TRUE; while (1) { rg_exec('virsh destroy ' . $ename, '', FALSE, FALSE); rg_exec('virsh undefine ' . $ename, '', FALSE, FALSE); $r = rg_del_tree($job['main']); if ($r === FALSE) { rg_log('Cannot delete main dir (' . $job['main'] . ')!'); break; } $r = @mkdir($job['main'], 0700); if ($r === FALSE) { rg_log('Cannot create main dir (' . $job['main'] . '):' . ' ' . $php_errormsg . '!'); break; } // Save & confirm the receiving (TODO: fsync) // TODO: This will be used to clean up on a restart $r = @file_put_contents($job['main'] . '/job.ser', serialize($job)); if ($r === FALSE) { $reason = 'cannot store job'; break; } $r = rg_exec('qemu-img create -b ' . $master . ' -f qcow2 ' . $img, '', FALSE, FALSE); if ($r['ok'] !== 1) { $reason = 'cannot create image: ' . $r['errmsg']; break; } $r = rg_exec('qemu-img create -f raw ' . $img2 . ' 1G', '', FALSE, FALSE); if ($r['ok'] !== 1) { $reason = 'cannot create image2: ' . $r['errmsg']; break; } // Seems that mkfs is not in PATH when it is runned from cron $path = getenv('PATH'); putenv('PATH=' . $path . ':/usr/sbin'); $r = rg_exec('mkfs.ext4 -L RG ' . $img2, '', FALSE, FALSE); if ($r['ok'] !== 1) { $reason = 'cannot create fs: ' . $r['errmsg']; break; } $r = @mkdir($job['main'] . '/root', 0700); if ($r === FALSE) { $reason = 'Cannot create root dir: ' . $php_errormsg . '!'; break; } $r = rg_exec('mount ' . $img2 . ' ' . $emain . '/root', '', FALSE, FALSE); if ($r['ok'] !== 1) { $reason = 'cannot mount fs: ' . $r['errmsg']; break; } $do_umount = TRUE; // Clone repo putenv('GIT_SSH_COMMAND=ssh' . ' -o PasswordAuthentication=no' . ' -o IdentityFile=' . escapeshellarg($conf['state'] . '/key')); $cmd = 'git clone --depth 1' . ' ' . escapeshellarg($job['url']) . ' ' . $emain . '/root/git'; $r = rg_exec($cmd, '', FALSE, FALSE); if ($r['ok'] !== 1) { $reason = 'cannot clone: ' . $r['errmsg']; break; } // Build command list // TODO: document how a user can add labels in configure or make $s = 'export RG_LABELS=/mnt/status/RG_LABELS' . "\n\n"; $s .= 'cd /mnt/git' . "\n\n"; $s .= 'git checkout ' . escapeshellarg($job['head']) . "\n"; foreach ($job['cmds'] as $name => $i) { if (empty($i['cmd'])) continue; $prefix = '/mnt/status/' . escapeshellarg($name); if (empty($i['label_ok'])) $lok = ''; else $lok = ' echo ' . escapeshellarg($i['label_ok']) . ' >>/mnt/status/RG_LABELS' . "\n"; if (empty($i['label_nok'])) $lnok = ''; else $lnok = ' echo ' . escapeshellarg($i['label_nok']) . ' >>/mnt/status/RG_LABELS' . "\n"; $s .= 'date +%s > ' . $prefix . '.start' . "\n" . '(' . $i['cmd'] . ') 1>' . $prefix . '.log 2>&1' . "\n" . 'E=${?}' . "\n" . 'date +%s > ' . $prefix . '.done' . "\n" . 'if [ "${E}" != "0" ]; then' . "\n" . ' echo ${E} > ' . $prefix . ".status\n" . $lnok . ($i['abort'] ? ' exit 0' . "\n" : '') . 'else' . "\n" . ' echo 0 > ' . $prefix . ".status\n" . $lok . 'fi' . "\n\n"; } $r = @file_put_contents($job['main'] . '/root/build.sh', $s); if ($r === FALSE) { $reason = 'cannot store build commands!'; break; } $r = @chmod($job['main'] . '/root/build.sh', 0700); if ($r === FALSE) { $reason = 'cannot to chmod on build.sh!'; break; } // Prepare packages - for now, we must list every package // on a single line to avoid not available packages $pkgs = explode(' ', $job['packages']); if (count($pkgs) > 0) { $p_i_cmd = ''; $p_i_cmd .= '> /mnt/packages.log' . "\n"; foreach ($pkgs as $p) { $p_i_cmd .= $env['pkg_cmd'] . ' ' . escapeshellarg($p) . ' >> /mnt/packages.log 2>&1' . "\n"; } } // Store commands $r = @file_put_contents($job['main'] . '/root/rg.sh', 'mkdir /mnt/status' . "\n" . 'chown -R build:build /mnt/git /mnt/status' . "\n" . 'date +%s > /mnt/T_START' . "\n" . '# Waiting for net...' . "\n" . 'while [ -z "`ip ro li | grep ^default`" ]; do' . "\n" . ' sleep 1' . "\n" . 'done' . "\n" . 'date +%s > /mnt/T_NET_OK' . "\n\n" . $p_i_cmd . 'date +%s > /mnt/T_PKGS_OK' . "\n\n" . 'su - build -c /mnt/build.sh' . "\n" . 'date +%s > /mnt/T_DONE' . "\n\n" . 'sync' . "\n" . 'shutdown -h now' ); if ($r === FALSE) { $reason = 'cannot store commands!'; break; } $r = @chmod($job['main'] . '/root/rg.sh', 0700); if ($r === FALSE) { $reason = 'cannot to chmod on rg.sh!'; break; } $r = rg_exec('umount ' . $emain . '/root', '', FALSE, FALSE); if ($r['ok'] !== 1) { $reason = 'cannot umount fs: ' . $r['errmsg']; break; } $do_umount = FALSE; // . ' --noautoconsole' // . ' --security type=dynamic,relabel=yes' // . ' --filesystem source=' . $emain . '/root') // . ',target=rg,mode=mapped' // passthrough // . ' --security type=static,label=' . $context . ',relabel=yes' $r = rg_exec('virt-install' . ' --name ' . $ename . ' --arch ' . escapeshellarg($env['arch']) . ' --memory 256' . ' --vcpus 1' . ' --graphics none' . ' --network network=default' . ' --security type=dynamic' . ' --os-variant ' . escapeshellarg($env['os-variant']) . ' --import' . ' --disk path=' . $img . ',discard=unmap' . ' --disk path=' . $img2 . ',discard=unmap' . ' --rng /dev/random' . ' --memballoon virtio' . ' --console pty,target_type=virtio' . ' ' . $env['paras'], '', FALSE, FALSE); if ($r['ok'] !== 1) { $reason = 'cannot define and start virtual machine: ' . $r['errmsg']; break; } $err = FALSE; break; } if ($do_umount) rg_exec('umount ' . $emain . '/root', '', FALSE, FALSE); // Seems that any error above must retrigger the build on other worker if ($err) @file_put_contents($job['main'] . '/error.log', $reason); } /* * Handle received commands */ function xhandle($key, $cmd0) { global $jobs; global $conf; global $pid_to_jid; $cmd = substr($cmd0, 0, 4); $data = stripcslashes(trim(substr($cmd0, 4))); $job = @unserialize($data); if ($job === FALSE) { rg_log('Cannot unserialize [' . $data . ']'); rg_conn_destroy($key); return; } $jid = $job['id']; if (strcmp($cmd, 'BLD ') == 0) { // TODO: should we confirm quickly if the job is accepted, // even if we could not fork? if (isset($jobs[$jid])) { // TODO: this should not happen, right? rg_log('Job ' . $jid . ' already in queue!'); return; } $jobs[$jid] = $job; $jobs[$jid]['main'] = $conf['state'] . '/rocketgit-j-' . $jid; $jobs[$jid]['state'] = RG_JOB_INIT; rg_log_ml('build job: ' . print_r($job, TRUE)); $err = TRUE; while (1) { // Fork $pid = pcntl_fork(); if ($pid == -1) { $reason = 'Cannot fork!'; break; } if ($pid == 0) { // child start_worker($jobs[$jid]); exit(0); } rg_log('Started worker with pid ' . $pid); $jobs[$jid]['state'] = RG_JOB_HELPER_STARTED; $pid_to_jid[$pid] = $jid; $err = FALSE; break; } $a = array('id' => $jid); if ($err) { $a['reason'] = $reason; rg_conn_enq('master', 'ABR ' . rg_conn_prepare($a) . "\n"); unset($pid_to_jid[$pid]); unset($jobs[$jid]); return; } rg_conn_enq('master', 'STA ' . rg_conn_prepare($a) . "\n"); } else if (strcmp($cmd, 'DRE ') == 0) { // DRE = done received // So, we can clean up everything related to this job // TODO: do we clear the state file? rg_log('DRE command'); $job = &$jobs[$jid]; unset($pid_to_jid[$job['pid']]); unset($jobs[$jid]); @unlink($job['main'] . '/job.ser'); } else { rg_log('Cannot handle[' . $key . ']: ' . $cmd); } } /* * Extracts info from the virtual disk * TODO: if something fails, we may keep the file mounted! */ function rg_job_extract_info(&$job) { global $conf; $jid = $job['id']; rg_log('DEBUG: extract_info: root=' . $job['main']); $emain = escapeshellarg($job['main']); while (1) { if (!is_dir($job['main'])) { $job['error'] = 'Main dir [' . $job['main'] . '] not presend;' . ' probably disk space problems'; break; } $r = @file_get_contents($job['main'] . '/error.log'); if ($r !== FALSE) { $job['error'] = $r; break; } $cmd = 'mount ' . $emain . '/image2.raw ' . $emain . '/root'; $r = rg_exec($cmd, '', FALSE, FALSE); if ($r['ok'] != 1) { $job['error'] = 'Could not mount image: ' . $r['errmsg']; break; } $labels = @file($job['main'] . '/root/status/RG_LABELS'); if ($labels === FALSE) $labels = array(); foreach ($labels as $index => $l) $labels[$index] = trim($l); // Add worker name as label $labels[] = 'worker/' . $conf['name'] . '/color=fff'; $job['status'] = array( 'packages' => @trim(file_get_contents($job['main'] . '/root/packages.log')), 'start' => @trim(file_get_contents($job['main'] . '/root/T_START')), 'net_ok' => @trim(file_get_contents($job['main'] . '/root/T_NET_OK')), 'pkgs_ok' => @trim(file_get_contents($job['main'] . '/root/T_PKGS_OK')), 'done' => @trim(file_get_contents($job['main'] . '/root/T_DONE')), 'labels' => $labels ); $job['status']['cmds'] = array(); foreach ($job['cmds'] as $cmd => $i) { if (empty($i['cmd'])) continue; $sd = $job['main'] . '/root/status/' . $cmd; $job['status']['cmds'][$cmd] = array( 'start' => @trim(file_get_contents($sd . '.start')), 'done' => @trim(file_get_contents($sd . '.done')), 'log' => @file_get_contents($sd . '.log', FALSE, NULL, -1, 4 * 4096) ); } unset($job['cmds']); unset($job['url']); unset($job['head']); unset($job['env']); $cmd = 'umount ' . $emain . '/root'; $r = rg_exec($cmd, '', FALSE, FALSE); if ($r['ok'] != 1) { rg_log('Cannot unmount: ' . $r['errmsg'] . '!'); break; } rg_del_tree($job['main']); break; } rg_log_ml('DEBUG: job: ' . print_r($job, TRUE)); return TRUE; } reload_config(); $socket = @socket_create(AF_INET, SOCK_STREAM, SOL_TCP); if ($socket === FALSE) { rg_log('Cannot create socket!'); exit(1); } $r = @socket_connect($socket, $conf['master'], $conf['port']); if ($r === FALSE) { rg_log('Cannot connect to ' . $conf['master'] . '/' . $conf['port'] . '!'); exit(1); } rg_conn_new('master', $socket); $rg_conns['master']['exit_on_close'] = 1; $rg_conns['master']['func_cmd'] = 'xhandle'; // announce ourselves $key = $conf['key']; unset($conf['key']); $ann = $conf; $ann['uname'] = php_uname('a'); $ann['host'] = php_uname('n'); $ann['arch'] = php_uname('m'); $ann['boot_time'] = time(); $ann['sign'] = hash_hmac('sha512', $ann['boot_time'], $key); rg_conn_enq('master', 'ANN ' . rg_conn_prepare($ann) . "\n"); $jobs = array(); $pid_to_jid = array(); while(1) { rg_log_buffer_clear(); rg_conn_wait(3); // Verify if the jobs are started while (1) { $pid = pcntl_waitpid(-1, $status, WNOHANG); if ($pid === 0) break; if ($pid == -1) break; $jid = $pid_to_jid[$pid]; rg_log('Pid ' . $pid . ' exited (job ' . $jid . ')' . ' with status ' . $status . '!'); unset($pid_to_jid[$pid]); $jobs[$jid]['state'] = RG_JOB_STARTED; } // Verify if vms finished $vms_loaded = FALSE; foreach ($jobs as $jid => &$job) { if ($job['state'] != RG_JOB_STARTED) continue; if ($vms_loaded === FALSE) { $vms = rg_builder_vm_list(); if ($vms === FALSE) break; $vms_loaded = TRUE; rg_log_ml('vms: ' . print_r($vms, TRUE)); } $name = 'rg-worker-' . $jid; $k = array_search($name, $vms); if ($k !== FALSE) { //rg_log('VM in progress'); // TODO: if too much time, abort (kill // worker and destroy virtual machine) //TODO: $job['error'] = 'too much time'; continue; } rg_log('VM ' . $jid . ' finished'); rg_job_extract_info($job); if (isset($job['error'])) $job['state'] = RG_JOB_ERROR; else $job['state'] = RG_JOB_DONE; // TODO: store in fs to be able to still inform the // master if we are crashing. @file_put_contents($job['main'] . '/job.ser', serialize($job)); $xjob = $job; unset($xjob['debug']); unset($xjob['packages']); unset($xjob['main']); rg_conn_enq('master', 'DON ' . rg_conn_prepare($xjob) . "\n"); // TODO: do we destroy the pool in case of crash? $cmd = 'virsh pool-destroy rocketgit-j-' . $jid; $r = rg_exec($cmd, '', FALSE, FALSE); if ($r['ok'] != 1) { $job['error'] = 'Could not destroy pool: ' . $r['errmsg']; rg_log('Error: ' . $job['error']); //break; TODO: do we need to do this?! } // TODO: do we clean the pool in case of crash? $cmd = 'virsh pool-undefine rocketgit-j-' . $jid; $r = rg_exec($cmd, '', FALSE, FALSE); if ($r['ok'] != 1) { $job['error'] = 'Could not undefine pool: ' . $r['errmsg']; rg_log('Error: ' . $job['error']); //break; TODO: do we need to do this?! } // TODO: do we clean the machine in case of crash? $cmd = 'virsh undefine rg-worker-' . escapeshellarg($jid); $r = rg_exec($cmd, '', FALSE, FALSE); if ($r['ok'] != 1) { $job['error'] = 'Could not undefine machine: ' . $r['errmsg']; rg_log('Error: ' . $job['error']); //break; TODO } } } rg_prof_end("MAIN"); rg_prof_log(); ?>