<?php // Client for continuous integration and deployment error_reporting(E_ALL); ini_set('track_errors', 'On'); set_time_limit(0); define('RG_JOB_INIT', 1); define('RG_JOB_HELPER_STARTED', 2); define('RG_JOB_STARTED', 3); define('RG_JOB_DONE', 10); $_s = microtime(TRUE); $INC = dirname(__FILE__) . "/../inc"; require_once($INC . "/init.inc.php"); require_once($INC . "/log.inc.php"); require_once($INC . "/prof.inc.php"); require_once($INC . "/builder.inc.php"); require_once($INC . "/conn.inc.php"); rg_prof_start('MAIN'); rg_log_set_file($rg_log_dir . '/worker.log'); rg_log_set_sid("000000"); // to spread the logs /* * Load configuration file */ function reload_config() { global $conf; $_conf = @file('/etc/rocketgit/worker.conf'); $last_key = FALSE; if ($_conf !== FALSE) { $conf = array('env' => array()); foreach ($_conf as $line) { if (empty(trim($line))) continue; if (strncmp($line, '#', 1) == 0) continue; $t = explode('=', $line, 2); if (count($t) != 2) { rg_log('Invalid line [' . $line . ']!'); continue; } $var = trim($t[0]); $value = trim($t[1]); if (strcmp($var, 'env') == 0) { $conf['env'][$value] = array('paras' => ''); $last_parent = &$conf['env'][$value]; } else if ((strncmp($line, " ", 1) == 0) || (strncmp($line, "\t", 1) == 0)) { if ($last_parent === FALSE) { rg_log('Invalid line [' . $line . ']!'); continue; } $t = explode('=', $line, 2); if (count($t) != 2) { rg_log('Invalid line [' . $line . ']!'); continue; } $var = trim($t[0]); $value = trim($t[1]); $last_parent[$var] = $value; } else { $conf[$var] = $value; $last_parent = FALSE; } } } else { $conf = array( 'workers' => '1', 'cost' => 1, 'env' => array() ); } rg_log_ml('conf: ' . print_r($conf, TRUE)); } /* * Starts an worker */ function start_worker($job) { global $conf; global $php_errormsg; $env = $conf['env'][$job['env']]; //rg_log_ml('DEBUG: env: ' . print_r($env, TRUE)); $jid = $job['id']; $emain = escapeshellarg($job['main']); $name = 'rg-worker-' . $job['id']; $ename = escapeshellarg($name); $master = escapeshellarg($env['image']); $img = escapeshellarg($job['main'] . '/image.qcow2'); $img2 = escapeshellarg($job['main'] . '/image2.raw'); $do_umount = FALSE; $err = TRUE; while (1) { rg_exec('virsh destroy ' . $ename); rg_exec('virsh undefine ' . $ename); $r = rg_del_tree($job['main']); if ($r === FALSE) { rg_log('Cannot delete main dir (' . $job['main'] . ')!'); break; } $r = @mkdir($job['main'], 0755); if ($r === FALSE) { rg_log('Cannot create main dir (' . $job['main'] . '):' . ' ' . $php_errormsg . '!'); break; } // Save & confirm the receiving (TODO: fsync) // TODO: This will be used to clean up on a restart $r = @file_put_contents($job['main'] . '/job.ser', serialize($job)); if ($r === FALSE) { $reason = 'cannot store job'; break; } $r = rg_exec('qemu-img create -b ' . $master . ' -f qcow2 ' . $img); if ($r['ok'] !== 1) { $reason = 'cannot create image: ' . $r['errmsg']; break; } $r = rg_exec('qemu-img create -f raw ' . $img2 . ' 1G'); if ($r['ok'] !== 1) { $reason = 'cannot create image2: ' . $r['errmsg']; break; } $r = rg_exec('mkfs.ext4 -L RG ' . $img2); if ($r['ok'] !== 1) { $reason = 'cannot create fs: ' . $r['errmsg']; break; } $r = @mkdir($job['main'] . '/root', 0755); if ($r === FALSE) { $reason = 'Cannot create root dir: ' . $php_errormsg . '!'; break; } $r = rg_exec('mount ' . $img2 . ' ' . $emain . '/root'); if ($r['ok'] !== 1) { $reason = 'cannot mount fs: ' . $r['errmsg']; break; } $do_umount = TRUE; // Clone repo $cmd = 'git clone --depth 1' . ' ' . escapeshellarg($job['url']) . ' ' . $emain . '/root/git'; $r = rg_exec($cmd); if ($r['ok'] !== 1) { $reason = 'cannot clone: ' . $r['errmsg']; break; } // Build command list $s = 'export RG_LABELS=/mnt/status/RG_LABELS' . "\n\n"; $s .= 'cd /mnt/git' . "\n\n"; $s .= 'git checkout ' . escapeshellarg($job['head']) . "\n"; foreach ($job['cmds'] as $name => $i) { $prefix = '/mnt/status/' . escapeshellarg($name); if (empty($i['label_ok'])) $lok = ''; else $lok = ' echo ' . escapeshellarg($i['label_ok']) . ' >>/mnt/status/RG_LABELS' . "\n"; if (empty($i['label_nok'])) $lnok = ''; else $lnok = ' echo ' . escapeshellarg($i['label_nok']) . ' >>/mnt/status/RG_LABELS' . "\n"; $s .= 'date +%s > ' . $prefix . '.start' . "\n" . '(' . $i['cmd'] . ') 1>' . $prefix . '.log 2>&1' . "\n" . 'E=${?}' . "\n" . 'date +%s > ' . $prefix . '.done' . "\n" . 'if [ "${E}" != "0" ]; then' . "\n" . ' echo ${E} > ' . $prefix . ".status\n" . $lnok . ($i['abort'] ? ' exit 0' . "\n" : '') . 'else' . "\n" . ' echo 0 > ' . $prefix . ".status\n" . $lok . 'fi' . "\n\n"; } $r = @file_put_contents($job['main'] . '/root/build.sh', $s); if ($r === FALSE) { $reason = 'cannot store build commands!'; break; } $r = @chmod($job['main'] . '/root/build.sh', 0755); if ($r === FALSE) { $reason = 'cannot to chmod on build.sh!'; break; } // Store commands $r = @file_put_contents($job['main'] . '/root/rg.sh', 'mkdir /mnt/status' . "\n" . 'chown -R build:build /mnt/git /mnt/status' . "\n" . 'date +%s > /mnt/T_START' . "\n" . 'su - build -c /mnt/build.sh' . "\n" . 'date +%s > /mnt/T_DONE' . "\n" . 'sync' . "\n" . 'shutdown -h now' ); if ($r === FALSE) { $reason = 'cannot store commands!'; break; } $r = @chmod($job['main'] . '/root/rg.sh', 0755); if ($r === FALSE) { $reason = 'cannot to chmod on rg.sh!'; break; } $r = rg_exec('umount ' . $emain . '/root'); if ($r['ok'] !== 1) { $reason = 'cannot umount fs: ' . $r['errmsg']; break; } $do_umount = FALSE; // . ' --console pty,target_type=virtio' // . ' --noautoconsole' // . ' --security type=dynamic,relabel=yes' // . ' --filesystem source=' . $emain . '/root') // . ',target=rg,mode=mapped' // passthrough // . ' --security type=static,label=' . $context . ',relabel=yes' $r = rg_exec('virt-install' . ' --name ' . $ename . ' --arch ' . escapeshellarg($env['arch']) . ' --memory 256' . ' --vcpus 1' . ' --graphics none' . ' --network none' . ' --security type=dynamic' . ' --os-variant fedora22' . ' --import' . ' --disk path=' . $img . ',discard=unmap' . ' --disk path=' . $img2 . ',discard=unmap' . ' --rng /dev/random' . ' ' . $env['paras']); if ($r['ok'] !== 1) { $reason = 'cannot define and start virtual machine: ' . $r['errmsg']; break; } $err = FALSE; break; } if ($do_umount) rg_exec('umount ' . $emain . '/root'); if ($err) @file_put_contents($job['main'] . '/error', $reason); } /* * Handle commands */ function xhandle($key, $cmd0) { global $jobs; global $conf; global $pid_to_jid; $cmd = substr($cmd0, 0, 4); $data = stripcslashes(trim(substr($cmd0, 4))); $job = @unserialize($data); if ($job === FALSE) { rg_log('Cannot unserialize [' . $data . ']'); rg_conn_destroy($key); return; } $jid = $job['id']; if (strcmp($cmd, 'BLD ') == 0) { if (isset($jobs[$jid])) { rg_log('Job ' . $jid . ' already in queue!'); return; } $jobs[$jid] = $job; $jobs[$jid]['main'] = $conf['state'] . '/j-' . $jid; $jobs[$jid]['state'] = RG_JOB_INIT; rg_log_ml('build job: ' . print_r($job, TRUE)); $err = TRUE; while (1) { // Fork $pid = pcntl_fork(); if ($pid == -1) { $reason = 'Cannot fork!'; break; } if ($pid == 0) { // child start_worker($jobs[$jid]); exit(0); } rg_log('Started builder with pid ' . $pid); $jobs[$jid]['state'] = RG_JOB_HELPER_STARTED; $pid_to_jid[$pid] = $jid; $err = FALSE; break; } $a = array('id' => $jid); if ($err) { $a['reason'] = $reason; rg_conn_enq('master', 'ABR ' . rg_conn_prepare($a) . "\n"); unset($pid_to_jid[$pid]); unset($jobs[$jid]); return; } rg_conn_enq('master', 'STA ' . rg_conn_prepare($a) . "\n"); } else if (strcmp($cmd, 'DRE ') == 0) { rg_log('DRE command'); $job = &$jobs[$jid]; unset($pid_to_jid[$job['pid']]); unset($jobs[$jid]); @unlink($job['main'] . '/job.ser'); } else { rg_log('Cannot handle[' . $key . ']: ' . $cmd); } } /* * Extracts info from the virtual disk */ function rg_job_extract_info(&$job) { global $conf; $jid = $job['id']; rg_log('DEBUG: extract_info: root=' . $job['main']); $emain = escapeshellarg($job['main']); while (1) { if (!is_dir($job['main'])) { $job['error'] = 'Main dir not presend;' . ' probably disk space problems'; break; } $r = @file_get_contents($job['main'] . '/error'); if ($r !== FALSE) { $job['error'] = $r; break; } $cmd = 'mount ' . $emain . '/image2.raw ' . $emain . '/root'; $r = rg_exec($cmd); if ($r['ok'] != 1) { $job['error'] = 'Could not mount image: ' . $r['data']; break; } $job['status'] = array( 'start' => @trim(file_get_contents($job['main'] . '/root/T_START')), 'done' => @trim(file_get_contents($job['main'] . '/root/T_DONE')), 'labels' => @file($job['main'] . '/root/status/RG_LABELS') ); foreach ($job['status']['labels'] as $index => $l) $job['status']['labels'][$index] = trim($l); $job['status']['cmds'] = array(); foreach ($job['cmds'] as $cmd => $i) { if (empty($i['cmd'])) continue; $sd = $job['main'] . '/root/status/' . $cmd; $job['status']['cmds'][$cmd] = array( 'start' => @trim(file_get_contents($sd . '.start')), 'done' => @trim(file_get_contents($sd . '.done')), 'log' => @file_get_contents($sd . '.log', FALSE, NULL, -1, 2048) ); } unset($job['cmds']); unset($job['url']); unset($job['head']); unset($job['env']); $cmd = 'umount ' . $emain . '/root'; $r = rg_exec($cmd); if ($r['ok'] != 1) { rg_log('Cannot unmount!'); break; } rg_del_tree($job['main']); break; } rg_log_ml('DEBUG: job: ' . print_r($job, TRUE)); return TRUE; } reload_config(); $socket = @socket_create(AF_INET, SOCK_STREAM, SOL_TCP); if ($socket === FALSE) { rg_log('Cannot create socket!'); exit(1); } $r = @socket_connect($socket, $conf['master'], $conf['port']); if ($r === FALSE) { rg_log('Cannot connect to ' . $conf['master'] . '/' . $conf['port'] . '!'); exit(1); } rg_conn_new('master', $socket); $rg_conns['master']['exit_on_close'] = 1; $rg_conns['master']['func_cmd'] = 'xhandle'; // announce ourselves $key = $conf['key']; unset($conf['key']); $ann = $conf; $ann['uname'] = php_uname('a'); $ann['host'] = php_uname('n'); $ann['arch'] = php_uname('m'); $ann['boot_time'] = time(); $ann['sign'] = hash_hmac('sha512', $ann['boot_time'], $key); rg_conn_enq('master', 'ANN ' . rg_conn_prepare($ann) . "\n"); $jobs = array(); $pid_to_jid = array(); while(1) { rg_log_buffer_clear(); rg_conn_wait(3); // Verify if the jobs are started while (1) { $pid = pcntl_waitpid(-1, $status, WNOHANG); if ($pid === 0) break; if ($pid == -1) break; $jid = $pid_to_jid[$pid]; rg_log('Pid ' . $pid . ' exited (job ' . $jid . ')!'); unset($pid_to_jid[$pid]); $jobs[$jid]['state'] = RG_JOB_STARTED; } // Verify if vms finished $vms_loaded = FALSE; foreach ($jobs as $jid => &$job) { if ($job['state'] != RG_JOB_STARTED) continue; if ($vms_loaded === FALSE) { $vms = rg_builder_vm_list(); if ($vms === FALSE) break; $vms_loaded = TRUE; rg_log_ml('vms: ' . print_r($vms, TRUE)); } $name = 'rg-worker-' . $jid; $k = array_search($name, $vms); if ($k === FALSE) { rg_log('VM finished'); $job['state'] = RG_JOB_DONE; rg_job_extract_info($job); // TODO: store in fs to be able to still inform the // master if we are crashing. $job['state'] = RG_JOB_DONE; @file_put_contents($job['main'] . '/job.ser', serialize($job)); rg_conn_enq('master', 'DON ' . rg_conn_prepare($job) . "\n"); } else { //rg_log('VM in progress'); // TODO: if too much time, abort } } } rg_prof_end("MAIN"); rg_prof_log(); ?>