status quo

author: Erich Eckner <git@eckner.net> 2018-04-11 16:54:13 +0200
committer: Erich Eckner <git@eckner.net> 2018-04-11 16:54:13 +0200
commit: 6d93913ca2893a7fc5499c0cb5d6e70831e7af50 (patch)
tree: 2e3b2e0a9779f75200648857852a9bc8171750ef
download: archweb32-6d93913ca2893a7fc5499c0cb5d6e70831e7af50.tar.xz
11 files changed, 1034 insertions, 0 deletions
diff --git a/index.html b/index.html
new file mode 100644
index 0000000..b5649b7
--- /dev/null
+++ b/index.html
@@ -0,0 +1,24 @@
+<html>
+<head>
+<title>Buildmaster for Archlinux32 packages</title>
+</head>
+<body>
+<a href="scripts/build-list.php">build list</a>
+<a href="scripts/build-list.php?show=broken">broken packages</a><br>
+<a href="scripts/build-slaves.php">build-slaves</a>
+<a href="scripts/status.php">status</a><br>
+<a href="build-logs/">build logs</a><br>
+<a href="namcap-outputs.html">namcap outputs</a><br>
+<a href="text-relocations-packages.html">packages with text relocations</a><br>
+<a href="mail-log.html">mail log</a><br>
+<a href="ssh-log.html">ssh log</a><br>
+sanity: of <a href="master-sanity.html">state files</a>,
+of <a href="mysql-sanity.html">mysql database</a>
+and <a href="scripts/mysql-issues.php">also some less critical stuff in the database</a><br>
+<a href="scripts/todos.php">todos</a>
+as <a href="scripts/todos.php?graph">graph</a><br>
+<a href="database-layout.png">database layout</a><br>
+<a href="scripts/blacklist.php">blacklisted packages</a><br>
+<img src="scripts/statistics.php?log"><br>
+</body>
+</html>
diff --git a/scripts/blacklist.php b/scripts/blacklist.php
new file mode 100644
index 0000000..c2810c6
--- /dev/null
+++ b/scripts/blacklist.php
@@ -0,0 +1,48 @@
+<html>
+<head>
+<title>Blacklisted packages</title>
+<link rel="stylesheet" type="text/css" href="/static/style.css">
+</head>
+<body>
+<table>
+<tr><th>architecture</th><th>package</th><th>reason</th></tr>
+<?php
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+if ( ! $result = $mysql -> query(
+  "SELECT DISTINCT `architectures`.`name` AS `architecture`,`package_sources`.`pkgbase`,`build_assignments`.`is_black_listed` " .
+  "FROM `build_assignments` " .
+  "JOIN `architectures` ON `build_assignments`.`architecture`=`architectures`.`id` " .
+  "JOIN `package_sources` ON `build_assignments`.`package_source`=`package_sources`.`id` " .
+  "WHERE `build_assignments`.`is_black_listed` IS NOT NULL " .
+  "ORDER BY `package_sources`.`pkgbase`")) {
+  die($mysql->error);
+}
+if ($result -> num_rows > 0) {
+  while($row = $result->fetch_assoc()) {
+    print "<tr><td>";
+    print $row["architecture"];
+    print "</td><td>";
+    print $row["pkgbase"];
+    print "</td><td>";
+    print preg_replace(
+      array (
+        "/FS32#(\\d+)/",
+        "/FS#(\\d+)/"
+      ),
+      array (
+        "<a href=\"https://bugs.archlinux32.org/index.php?do=details&task_id=$1\">$0</a>",
+        "<a href=\"https://bugs.archlinux.org/task/$1\">$0</a>"
+      ),
+      $row["is_black_listed"]
+    );
+    print "</td></tr>\n";
+  }
+}
+?>
+</table>
+</body>
+</html>
diff --git a/scripts/build-list.php b/scripts/build-list.php
new file mode 100644
index 0000000..5cc7a47
--- /dev/null
+++ b/scripts/build-list.php
@@ -0,0 +1,220 @@
+<html>
+<head>
+<?php
+
+if (isset($_GET["show"]))
+  $to_show=$_GET["show"];
+else
+  $to_show="all";
+
+if ($to_show == "all")
+  $match = "";
+elseif ($to_show == "broken")
+  $match = " AND (`build_assignments`.`is_broken` OR `build_assignments`.`is_blocked` IS NOT NULL)";
+elseif ($to_show == "next")
+  $match = "";
+else
+  die();
+
+print "<title>List of " . $to_show . " package builds</title>\n";
+print "<link rel=\"stylesheet\" type=\"text/css\" href=\"/static/style.css\">\n";
+print "</head>\n";
+print "<body>\n";
+print "<a href=\"/\">Start page</a>\n";
+print "<a href=\"/build-logs/\">build logs</a><br>\n";
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+
+$result = $mysql -> query(
+  "SELECT DISTINCT " .
+  "`build_assignments`.`id`," .
+  "`build_assignments`.`is_blocked`," .
+  "`package_sources`.`pkgbase`," .
+  "`package_sources`.`git_revision`," .
+  "`package_sources`.`mod_git_revision`," .
+  "`package_sources`.`uses_upstream`," .
+  "`package_sources`.`uses_modification`," .
+  "`upstream_repositories`.`name` AS `package_repository`," .
+  "`git_repositories`.`name` AS `git_repository`," .
+  "`architectures`.`name` AS `arch`," .
+  "EXISTS (SELECT * " .
+    "FROM `binary_packages` `broken_bin` " .
+    "JOIN `dependencies` ON `dependencies`.`dependent` = `broken_bin`.`id` " .
+    "JOIN `install_target_providers` ON `install_target_providers`.`install_target` = `dependencies`.`depending_on` " .
+    "JOIN `binary_packages` `to_be_built` ON `to_be_built`.`id` = `install_target_providers`.`package` " .
+    "JOIN `repositories` ON `to_be_built`.`repository` = `repositories`.`id` " .
+    "WHERE `broken_bin`.`build_assignment`=`build_assignments`.`id` ".
+    "AND `repositories`.`name`=\"build-list\" " .
+    "AND `to_be_built`.`build_assignment`!=`build_assignments`.`id`" .
+  ") AS `dependencies_pending`," .
+  "(SELECT count(*) " .
+    "FROM `build_dependency_loops` " .
+    "WHERE `build_dependency_loops`.`build_assignment`=`build_assignments`.`id`" .
+  ") AS `loops`, " .
+  "`build_slaves`.`name` AS `build_slave` " .
+  "FROM `build_assignments` " .
+  "JOIN `architectures` ON `build_assignments`.`architecture` = `architectures`.`id` " .
+  "JOIN `package_sources` ON `build_assignments`.`package_source` = `package_sources`.`id` " .
+  "JOIN `upstream_repositories` ON `package_sources`.`upstream_package_repository` = `upstream_repositories`.`id` " .
+  "JOIN `git_repositories` ON `upstream_repositories`.`git_repository`=`git_repositories`.`id` " .
+  "JOIN `binary_packages` ON `binary_packages`.`build_assignment` = `build_assignments`.`id` " .
+  "JOIN `repositories` ON `binary_packages`.`repository` = `repositories`.`id` " .
+  "LEFT JOIN `build_slaves` ON `build_slaves`.`currently_building`=`build_assignments`.`id` " .
+  "WHERE `repositories`.`name`=\"build-list\"" . $match
+);
+if ($result -> num_rows > 0) {
+
+  $count = 0;
+
+  while($row = $result->fetch_assoc()) {
+
+    if (($to_show == "next") &&
+      ($row["loops"]==0) &&
+      ($row["dependencies_pending"]==1))
+      continue;
+
+    $fail_result = $mysql -> query(
+      "SELECT " .
+      "`fail_reasons`.`name`, " .
+      "`failed_builds`.`log_file` " .
+      "FROM `failed_builds` " .
+      "JOIN `fail_reasons` ON `failed_builds`.`reason`=`fail_reasons`.`id` " .
+      "WHERE `failed_builds`.`build_assignment`=".$row["id"]." " .
+      "ORDER BY `failed_builds`.`date`"
+    );
+
+    unset($reasons);
+    $rows[$count]["trials"] = $fail_result -> num_rows;
+    if ($rows[$count]["trials"] > 0) {
+      while($fail_row = $fail_result->fetch_assoc()) {
+        $reasons[$fail_row["name"]] = $fail_row["log_file"];
+      }
+    }
+    if (isset($reasons)) {
+      $to_print="";
+      foreach ($reasons as $reason => $last_log) {
+        if (file_exists("/srv/http/build-logs/error/".$last_log)) {
+          $to_print= $to_print .
+            ", <a href=\"/build-logs/error/" .
+            $last_log .
+            "\">" .
+            $reason .
+            "</a>";
+        } else {
+          $to_print= $to_print . ", " . $reason;
+        }
+      }
+      $rows[$count]["fail_reasons"]=substr($to_print,2);
+    } else {
+      $rows[$count]["fail_reasons"]="&nbsp;";
+    }
+
+    $rows[$count]["loops"] = $row["loops"];
+    $rows[$count]["pkgbase"] = $row["pkgbase"];
+    if ($row["dependencies_pending"]=="0")
+      $rows[$count]["pkgbase_print"] = $rows[$count]["pkgbase"];
+    else
+      $rows[$count]["pkgbase_print"] = "(" . $rows[$count]["pkgbase"] . ")";
+    if ($row["uses_upstream"]) {
+      $rows[$count]["git_revision"] =
+        "<a href=\"https://git.archlinux.org/svntogit/" .
+        $row["git_repository"] . ".git/tree/" .
+        $row["pkgbase"] . "/repos/" .
+        $row["package_repository"] . "-";
+      if ($row["arch"]=="any")
+        $rows[$count]["git_revision"] =
+          $rows[$count]["git_revision"] . "any";
+      else
+        $rows[$count]["git_revision"] =
+          $rows[$count]["git_revision"] . "x86_64";
+      $rows[$count]["git_revision"] =
+        $rows[$count]["git_revision"] . "?id=" .
+        $row["git_revision"];
+      $rows[$count]["git_revision"] =
+        $rows[$count]["git_revision"] . "\">" .
+        $row["git_revision"] . "</a>";
+    } else
+      $rows[$count]["git_revision"] = $row["git_revision"];
+    if ($row["uses_modification"])
+      $rows[$count]["mod_git_revision"] =
+        "<a href=\"https://github.com/archlinux32/packages/tree/" .
+        $row["mod_git_revision"] . "/" .
+        $row["package_repository"] . "/" .
+        $row["pkgbase"] . "\">" .
+        $row["mod_git_revision"] . "</a>";
+    else
+      $rows[$count]["mod_git_revision"] = $row["mod_git_revision"];
+    $rows[$count]["package_repository"] = $row["package_repository"];
+    if ($row["is_blocked"]=="") {
+      $rows[$count]["is_blocked"]="&nbsp;";
+    }
+    else {
+      $rows[$count]["is_blocked"] = preg_replace(
+        array (
+          "/FS32#(\\d+)/",
+          "/FS#(\\d+)/"
+        ),
+        array (
+          "<a href=\"https://bugs.archlinux32.org/index.php?do=details&task_id=$1\">$0</a>",
+          "<a href=\"https://bugs.archlinux.org/task/$1\">$0</a>"
+        ),
+        $row["is_blocked"]
+      );
+    }
+    if (isset($row["build_slave"]))
+      $rows[$count]["build_slave"] = $row["build_slave"];
+    else
+      $rows[$count]["build_slave"] = "&nbsp;";
+    $count++;
+  }
+
+  usort(
+    $rows,
+    function (array $a, array $b) {
+      if ($a["trials"] < $b["trials"])
+        return -1;
+      if ($a["trials"] > $b["trials"])
+        return 1;
+      return strcmp($a["pkgbase"],$b["pkgbase"]);
+    }
+  );
+
+  print "<table>\n";
+  print "<tr>";
+  print "<th>package</th>";
+  print "<th>git revision</th>";
+  print "<th>modification git revision</th>";
+  print "<th>package repository</th>";
+  print "<th>compilations</th>";
+  print "<th>loops</th>";
+  print "<th>build error</th>";
+  print "<th>blocked</th>";
+  print "<th>handed out to</th>";
+  print "</tr>\n";
+
+  foreach($rows as $row) {
+
+    print "<tr>";
+
+    print "<td><a href=\"/scripts/dependencies.php?b=".$row["pkgbase"]."&r=build-list\">".$row["pkgbase_print"]."</a></td>";
+    print "<td><p style=\"font-size:8px\">".$row["git_revision"]."</p></td>";
+    print "<td><p style=\"font-size:8px\">".$row["mod_git_revision"]."</p></td>";
+    print "<td>".$row["package_repository"]."</td>";
+    print "<td>".$row["trials"]."</td>";
+    print "<td>".$row["loops"]."</td>";
+    print "<td>".$row["fail_reasons"]."</td>";
+    print "<td>".$row["is_blocked"]."</td>";
+    print "<td>".$row["build_slave"]."</td>";
+
+    print "</tr>\n";
+  }
+
+  print "</table>\n";
+}
+
+?>
+</body>
+</html>
diff --git a/scripts/build-slaves.php b/scripts/build-slaves.php
new file mode 100644
index 0000000..b6afc2d
--- /dev/null
+++ b/scripts/build-slaves.php
@@ -0,0 +1,49 @@
+<html><head><title>list of build slaves</title></head><body>
+<?php
+
+  $conn = new mysqli("localhost","webserver","empty","buildmaster");
+  if ($conn->connect_error) {
+    die("Connection to mysql database failed: " . $conn->connect_error);
+  }
+
+  if (!$result =
+    $conn->query(
+      "SELECT" .
+      " `build_slaves`.`name`," .
+      "`build_slaves`.`operator`," .
+      "`package_sources`.`pkgbase`," .
+      "`build_slaves`.`last_connection`," .
+      "`build_slaves`.`logged_lines`," .
+      "`build_slaves`.`last_action`" .
+      " FROM `build_slaves`" .
+      " LEFT JOIN `build_assignments` ON" .
+      " `build_slaves`.`currently_building`=`build_assignments`.`id`" .
+      " LEFT JOIN `package_sources` ON" .
+      " `build_assignments`.`package_source`=`package_sources`.`id`" .
+      " ORDER BY `build_slaves`.`last_connection`"
+    ))
+    die($conn->error);
+
+  print "<table border=1>\n";
+  if ($result->num_rows > 0) {
+    print "<tr><th>name</th><th>operator</th><th>currently building</th><th>last connection</th><th>logged lines</th><th>last action</th></tr>\n";
+    while ($row = $result -> fetch_assoc()) {
+      foreach ($row as $key => $value) {
+        if ($value=="") {
+          $row[$key]="&nbsp;";
+        }
+      }
+      print "<tr>";
+      print "<td>".$row["name"]."</td>";
+      print "<td>".$row["operator"]."</td>";
+      print "<td>".$row["pkgbase"]."</td>";
+      print "<td>".$row["last_connection"]."</td>";
+      print "<td>".$row["logged_lines"]."</td>";
+      print "<td>".$row["last_action"]."</td>";
+      print "</tr>\n";
+    }
+  }
+  print "</table>\n";
+
+?>
+</body></html>
diff --git a/scripts/dependencies.php b/scripts/dependencies.php
new file mode 100644
index 0000000..3a38ac8
--- /dev/null
+++ b/scripts/dependencies.php
@@ -0,0 +1,179 @@
+<?php
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+
+$match = "";
+
+if (isset($_GET["a"]))
+  $match .= " AND `architectures`.`name`=from_base64(\"" . base64_encode($_GET["a"]) . "\")";
+if (isset($_GET["b"]))
+  $match .= " AND `package_sources`.`pkgbase`=from_base64(\"" . base64_encode($_GET["b"]) . "\")";
+if (isset($_GET["p"]))
+  $match .= " AND `binary_packages`.`pkgname`=from_base64(\"" . base64_encode($_GET["p"]) . "\")";
+if (isset($_GET["r"]))
+  $match .= " AND `repositories`.`name`=from_base64(\"" . base64_encode($_GET["r"]) . "\")";
+
+$ignore_install_targets = " AND NOT `install_targets`.`name` IN (\"base\",\"base-devel\")";
+
+$colors["stable"]="#000000";
+$colors["testing"]="#008000";
+$colors["staging"]="#00ff00";
+$colors["standalone"]="#000000";
+$colors["unbuilt"]="#ff0000";
+$colors["forbidden"]="#808080";
+$colors["virtual"]="#800080";
+
+$limit=200;
+
+if (! $result = $mysql -> query(
+  "CREATE TEMPORARY TABLE `cons` (" .
+    "`dep` BIGINT, " .
+    "`itp` BIGINT, " .
+    "UNIQUE KEY `content` (`dep`,`itp`)" .
+  ")"))
+  die($mysql->error);
+
+if (! $result = $mysql -> query(
+  "INSERT IGNORE INTO `cons` (`dep`,`itp`)" .
+  " SELECT `dependencies`.`id`,`install_target_providers`.`id`".
+  " FROM `binary_packages`" .
+  " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
+  " JOIN `repository_stabilities` ON `repositories`.`stability`=`repository_stabilities`.`id`" .
+  " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+  " JOIN `build_assignments` ON `binary_packages`.`build_assignment`= `build_assignments`.`id`" .
+  " JOIN `package_sources` ON `build_assignments`.`package_source`= `package_sources`.`id`" .
+  $match .
+  " JOIN `dependencies` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
+  " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`" .
+  " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
+  $ignore_install_targets .
+  " JOIN `install_target_providers` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`" .
+  " WHERE (`dependency_types`.`relevant_for_binary_packages` OR `repository_stabilities`.`name`=\"unbuilt\")" .
+  " LIMIT " . $limit
+  ))
+  die($mysql->error);
+
+if (! $result = $mysql -> query(
+  "INSERT IGNORE INTO `cons` (`dep`,`itp`)" .
+  " SELECT `dependencies`.`id`,`install_target_providers`.`id`".
+  " FROM `binary_packages`" .
+  " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
+  " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+  " JOIN `build_assignments` ON `binary_packages`.`build_assignment`= `build_assignments`.`id`" .
+  " JOIN `package_sources` ON `build_assignments`.`package_source`= `package_sources`.`id`" .
+  $match .
+  " JOIN `install_target_providers` ON `install_target_providers`.`package`=`binary_packages`.`id`" .
+  " JOIN `dependencies` ON `install_target_providers`.`install_target`=`dependencies`.`depending_on`" .
+  " JOIN `binary_packages` AS `d_bp` ON `dependencies`.`dependent`=`d_bp`.`id`" .
+  " JOIN `repositories` AS `d_r` ON `d_bp`.`repository`=`d_r`.`id`" .
+  " JOIN `repository_stabilities` AS `d_rs` ON `d_r`.`stability`=`d_rs`.`id`" .
+  " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`" .
+  " WHERE (`dependency_types`.`relevant_for_binary_packages` OR `d_rs`.`name`=\"unbuilt\")" .
+  " LIMIT " . $limit
+  ))
+  die($mysql->error);
+
+unset($knots);
+unset($edges);
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT `install_target_providers`.`install_target`,`install_target_providers`.`package`" .
+  " FROM `cons`" .
+  " JOIN `install_target_providers` ON `cons`.`itp`=`install_target_providers`.`id`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $edges .= "\"p" . $row["package"] . "\" -> \"i" . $row["install_target"] . "\" [color = \"#000080\"];\n";
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT `dependencies`.`dependent`,`dependencies`.`depending_on`,`dependency_types`.`name`" .
+  " FROM `cons`" .
+  " JOIN `dependencies` ON `cons`.`dep`=`dependencies`.`id`" .
+  " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $edges .= "\"i" . $row["depending_on"] . "\" -> \"p" . $row["dependent"] . "\" [taillabel = \"" . $row["name"] . "\"];\n";
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT `install_targets`.`id`,`install_targets`.`name`" .
+  " FROM `cons`" .
+  " JOIN `dependencies` ON `cons`.`dep`=`dependencies`.`id`" .
+  " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $knots .= "\"i" . $row["id"] . "\" [label = \"" . $row["name"] . "\", fontcolor = \"#000080\"];\n";
+
+$pkgfile_query =
+  "CONCAT(".
+    "`repositories`.`name`,\"/\"," .
+    "`binary_packages`.`pkgname`,\"-\"," .
+    "IF(`binary_packages`.`epoch`=0,\"\",CONCAT(`binary_packages`.`epoch`,\":\"))," .
+    "`binary_packages`.`pkgver`,\"-\"," .
+    "`binary_packages`.`pkgrel`,\".\"," .
+    "`binary_packages`.`sub_pkgrel`,\"-\"," .
+    "`architectures`.`name`" .
+  ") AS `filename`";
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT " .
+  "`binary_packages`.`id`," .
+  "`repository_stabilities`.`name` AS `stability`," .
+  $pkgfile_query .
+  " FROM `cons`" .
+  " JOIN `dependencies` ON `cons`.`dep`=`dependencies`.`id`" .
+  " JOIN `binary_packages` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
+  " JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`" .
+  " JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`" .
+  " JOIN `repository_stabilities` ON `repository_stabilities`.`id`=`repositories`.`stability`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $knots .= "\"p" . $row["id"] . "\" [label = \"" . $row["filename"] . "\", fontcolor = \"" . $colors[$row["stability"]] . "\"];\n";
+
+if (! $result = $mysql -> query(
+  "SELECT DISTINCT " .
+  "`binary_packages`.`id`," .
+  "`repository_stabilities`.`name` AS `stability`," .
+  $pkgfile_query .
+  " FROM `cons`" .
+  " JOIN `install_target_providers` ON `cons`.`itp`=`install_target_providers`.`id`" .
+  " JOIN `binary_packages` ON `install_target_providers`.`package`=`binary_packages`.`id`" .
+  " JOIN `architectures` ON `architectures`.`id`=`binary_packages`.`architecture`" .
+  " JOIN `repositories` ON `repositories`.`id`=`binary_packages`.`repository`" .
+  " JOIN `repository_stabilities` ON `repository_stabilities`.`id`=`repositories`.`stability`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0)
+  while ($row = $result->fetch_assoc())
+    $knots .= "\"p" . $row["id"] . "\" [label = \"" . $row["filename"] . "\", fontcolor = \"" . $colors[$row["stability"]] . "\"];\n";
+
+$knots = str_replace("\$","\\\$",$knots);
+$edges = str_replace("\$","\\\$",$edges);
+
+header ("Content-type: image/png");
+passthru(
+  "dot -Tpng -o/dev/stdout /dev/stdin <<EOF\n" .
+  "digraph dependencies {\n" .
+  "rankdir=LR;\n" .
+  "fontname=dejavu;\n" .
+  $knots .
+  $edges .
+  "}\n" .
+  "EOF\n"
+);
+
+?>
diff --git a/scripts/mysql-issues.php b/scripts/mysql-issues.php
new file mode 100644
index 0000000..2f1d99d
--- /dev/null
+++ b/scripts/mysql-issues.php
@@ -0,0 +1,136 @@
+<html>
+<head>
+<title>More and less critical issues with the database</title>
+<link rel="stylesheet" type="text/css" href="/static/style.css">
+</head>
+<body>
+<a href="/">Start page</a><br>
+<?php
+
+  if (isset($_GET["ignore-haskell"]))
+    $ignore = " AND `install_targets`.`name` NOT LIKE \"libHS%\"";
+  else
+    $ignore = "";
+
+  $mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+  if ($mysql->connect_error) {
+    die("Connection failed: " . $mysql->connect_error);
+  }
+
+  if (! $result = $mysql -> query(
+    "SELECT CONCAT(" .
+    "`repositories`.`name`,\"/\"," .
+    "`binary_packages`.`pkgname`,\"-\"," .
+    "IF(`binary_packages`.`epoch`=0,\"\",CONCAT(`binary_packages`.`epoch`,\":\"))," .
+    "`binary_packages`.`pkgver`,\"-\"," .
+    "`binary_packages`.`pkgrel`,\".\"," .
+    "`binary_packages`.`sub_pkgrel`,\"-\"," .
+    "`architectures`.`name`) AS `pkgfile`," .
+    "`install_targets`.`name` AS `install_target`," .
+    "IF(`binary_packages`.`is_to_be_deleted`,1,0) AS `is_to_be_deleted`," .
+    "`subst_r`.`name` AS `subst_repository`," .
+    "`subst_buildlist_bp`.`id` AS `subst_buildlist`" .
+    " FROM `binary_packages`" .
+    " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
+    " AND `repositories`.`is_on_master_mirror`" .
+    " JOIN `dependencies` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
+    " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`" .
+    " AND `dependency_types`.`relevant_for_binary_packages`" .
+    " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
+    " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+    " LEFT JOIN (`binary_packages` AS `subst_bp`" .
+    " JOIN `repositories` AS `subst_r` ON `subst_bp`.`repository`=`subst_r`.`id`" .
+    " JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`subst_r`.`id`" .
+    ")" .
+    " ON `subst_bp`.`pkgname`=`binary_packages`.`pkgname`" .
+    " AND `subst_bp`.`id`!=`binary_packages`.`id`" .
+    " AND `repository_stability_relations`.`more_stable`=`repositories`.`id`" .
+    " LEFT JOIN (`binary_packages` AS `subst_buildlist_bp`" .
+    " JOIN `repositories` AS `subst_buildlist_r`" .
+    " ON `subst_buildlist_bp`.`repository`=`subst_buildlist_r`.`id`" .
+    " AND `subst_buildlist_r`.`name`=\"build-list\"".
+    ") ON `subst_buildlist_bp`.`pkgname`=`binary_packages`.`pkgname`" .
+    " WHERE NOT EXISTS (" .
+      "SELECT * FROM `install_target_providers`" .
+      " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
+    ")" .
+    $ignore .
+    " ORDER BY `is_to_be_deleted`, `binary_packages`.`pkgname`"
+  ))
+    die($mysql -> error);
+
+  print "Found " . ($result -> num_rows) . " serious issues.<br>\n";
+
+  if ($result -> num_rows > 0) {
+
+    while ($row = $result->fetch_assoc()) {
+      if ($row["is_to_be_deleted"]==1)
+        print "<font color=\"#00ff00\">(marked as to-be-deleted) ";
+      else
+        print "<font color=\"#ff0000\">";
+      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package";
+      if (isset($row["subst_repository"]))
+        print " - but can be replaced by the one in " . $row["subst_repository"];
+      elseif (isset($row["subst_buildlist"]))
+        print " - but is already rescheduled";
+      print ".<br>";
+      print "</font>\n";
+    }
+
+  }
+
+  if (! $result = $mysql -> query(
+    "SELECT CONCAT(" .
+    "`repositories`.`name`,\"/\"," .
+    "`binary_packages`.`pkgname`,\"-\"," .
+    "IF(`binary_packages`.`epoch`=0,\"\",CONCAT(`binary_packages`.`epoch`,\":\"))," .
+    "`binary_packages`.`pkgver`,\"-\"," .
+    "`binary_packages`.`pkgrel`,\".\"," .
+    "`binary_packages`.`sub_pkgrel`,\"-\"," .
+    "`architectures`.`name`) AS `pkgfile`," .
+    "`install_targets`.`name` AS `install_target`," .
+    "`repository_stabilities`.`name` AS `stability`," .
+    "IF(`binary_packages`.`is_to_be_deleted`,1,0) AS `is_to_be_deleted`" .
+    " FROM `binary_packages`" .
+    " JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id`" .
+    " AND `repositories`.`is_on_master_mirror`" .
+    " JOIN `repository_stabilities` ON `repositories`.`stability`=`repository_stabilities`.`id`" .
+    " JOIN `dependencies` ON `dependencies`.`dependent`=`binary_packages`.`id`" .
+    " JOIN `dependency_types` ON `dependencies`.`dependency_type`=`dependency_types`.`id`" .
+    " AND `dependency_types`.`relevant_for_binary_packages`" .
+    " JOIN `install_targets` ON `dependencies`.`depending_on`=`install_targets`.`id`" .
+    " JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id`" .
+    " WHERE EXISTS (" .
+      "SELECT * FROM `install_target_providers`" .
+      " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
+    ")" .
+    " AND NOT EXISTS (" .
+      "SELECT * FROM `install_target_providers`" .
+      " JOIN `binary_packages` AS `prov_bp` ON `prov_bp`.`id`=`install_target_providers`.`package`" .
+      " JOIN `repositories` AS `prov_r` ON `prov_bp`.`repository`=`prov_r`.`id`" .
+      " JOIN `repository_stability_relations` ON `prov_r`.`stability`=`repository_stability_relations`.`more_stable`" .
+      " WHERE `install_target_providers`.`install_target` = `dependencies`.`depending_on`" .
+      " AND `repositories`.`stability`=`repository_stability_relations`.`less_stable`" .
+    ")" .
+    $ignore .
+    " ORDER BY `is_to_be_deleted`, `binary_packages`.`pkgname`"
+  ))
+    die($mysql -> error);
+
+  print "Found " . ($result -> num_rows) . " stability issues.<br>\n";
+
+  if ($result -> num_rows > 0) {
+    while ($row = $result->fetch_assoc()) {
+      if ($row["is_to_be_deleted"]==1)
+        print "<font color=\"#00ff00\">(marked as to-be-deleted) ";
+      else
+        print "<font color=\"#800000\">";
+      print $row["pkgfile"] . " depends on " . $row["install_target"] . " which is not provided by any package installable from enabled " . $row["stability"] . " repositories.<br>";
+      print "</font>\n";
+    }
+
+  }
+
+?>
+</body>
+</html>
diff --git a/scripts/packages.php b/scripts/packages.php
new file mode 100644
index 0000000..b7c0196
--- /dev/null
+++ b/scripts/packages.php
@@ -0,0 +1,20 @@
+<?php
+
+  $mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+  if ($mysql->connect_error) {
+    die("Connection to mysql database failed: " . $mysql->connect_error);
+  }
+
+  $result = $mysql -> query("SELECT * FROM `binary_packages`");
+  if ($result -> num_rows > 0) {
+    while($row = $result->fetch_assoc()) {
+      foreach ($row as $key => $val) {
+        print $key .": ".$val." - ";
+      }
+      print "<br>\n";
+    }
+  }
+
+  print 'OK';
+
+?>
diff --git a/scripts/statistics.php b/scripts/statistics.php
new file mode 100644
index 0000000..c919f93
--- /dev/null
+++ b/scripts/statistics.php
@@ -0,0 +1,172 @@
+<?php
+
+if (isset($_GET["from"]))
+  $min_time="from_base64(\"" . base64_encode("-".$_GET["from"]) . "\")";
+else
+  $min_time="\"-7 00:00:00\"";
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+if (! $result = $mysql -> query(
+    "SELECT DISTINCT ".
+    "UNIX_TIMESTAMP(`statistics`.`date`) AS `date`," .
+    "`statistics`.`pending_tasks_count`," .
+    "`statistics`.`pending_packages_count`," .
+    "`statistics`.`staging_packages_count`," .
+    "`statistics`.`testing_packages_count`," .
+    "`statistics`.`tested_packages_count`," .
+    "`statistics`.`broken_tasks_count`," .
+    "`statistics`.`dependency_loops_count`," .
+    "`statistics`.`dependency_looped_tasks_count`," .
+    "`statistics`.`locked_tasks_count`," .
+    "`statistics`.`blocked_tasks_count`," .
+    "`statistics`.`next_tasks_count`" .
+    "FROM `statistics` " .
+    "WHERE `statistics`.`date`>=ADDTIME(NOW()," . $min_time . ") " .
+    "ORDER BY `statistics`.`date`"
+  ))
+  die($mysql->error);
+
+$t_min = -1;
+$t_max = -1;
+$val_max = -1;
+
+while($vals = $result->fetch_assoc()) {
+  if ($t_min == -1)
+    $t_min = $vals["date"];
+  $t_max = $vals["date"];
+  foreach ($vals as $column => $val)
+    if ($column != "date") {
+      $values[$column][$vals["date"]] = $val;
+      $val_max = max($val_max,$val);
+    }
+};
+$print_columns = array_keys($values);
+
+$max_len = 0;
+foreach ($print_columns as $column) {
+  $len = strlen($values[$column][$t_max])+1;
+  if ($len > $max_len)
+    $max_len = $len;
+}
+
+$width = 1600;
+$height = 600;
+$border = 5;
+$legend_line_length = 10;
+$legend_height = 4 * ImageFontHeight(5) + $legend_line_length;
+
+$im = @ImageCreate ($width + $legend_line_length + $max_len * ImageFontWidth(5), $height + $legend_height)
+      or die ("Cannot create new gd-image-stream");
+
+$background_color = ImageColorAllocate ($im, 255, 255, 255);
+$foreground_color = ImageColorAllocate ($im, 0, 0, 0);
+
+$colors['stable_packages_count'] = ImageColorAllocate ($im, 0, 0, 0);
+$colors['pending_tasks_count'] = ImageColorAllocate ($im, 0, 0, 128);
+$colors['pending_packages_count'] = ImageColorAllocate ($im, 0, 0, 255);
+$colors['staging_packages_count'] = ImageColorAllocate ($im, 0, 100, 0);
+$colors['testing_packages_count'] = ImageColorAllocate ($im, 0, 200, 0);
+$colors['tested_packages_count'] = ImageColorAllocate ($im, 100, 255, 0);
+$colors['broken_tasks_count'] = ImageColorAllocate ($im, 255, 0, 0);
+$colors['dependency_loops_count'] = ImageColorAllocate ($im, 128, 128, 0);
+$colors['dependency_looped_tasks_count'] = ImageColorAllocate ($im, 255, 128, 128);
+$colors['locked_tasks_count'] = ImageColorAllocate ($im, 128, 128, 128);
+$colors['blocked_tasks_count'] = ImageColorAllocate ($im, 128, 0, 0);
+$colors['next_tasks_count'] = ImageColorAllocate ($im, 0, 255, 255);
+
+function scale($x, $x_min, $x_max, $scale, $log) {
+  if ($log) {
+    $x = log($x + 10);
+    $x_min = log($x_min + 10);
+    $x_max = log($x_max + 10);
+  };
+  if ($x_max == $x_min)
+    $frac = 0;
+  else
+    $frac = ($x - $x_min)/($x_max - $x_min);
+  if ($scale < 0)
+    return ($frac-1) * $scale;
+  else
+    return $frac * $scale;
+};
+
+function print_graph($data, $color) {
+  global $width, $height, $im, $t_min, $t_max, $val_max, $border, $legend_line_length;
+  ksort($data);
+  $last_t = -1;
+  $last_val = -1;
+  foreach ($data as $t => $val) {
+    if ($last_t != -1)
+      ImageLine(
+        $im,
+        scale($last_t,$t_min,$t_max,$width-2*$border,false)+$border+$legend_line_length,
+        scale($last_val,0,$val_max,-$height+2*$border,isset($_GET["log"]))+$border,
+        scale($t,$t_min,$t_max,$width-2*$border,false)+$border+$legend_line_length,
+        scale($val,0,$val_max,-$height+2*$border,isset($_GET["log"]))+$border,
+        $color
+      );
+    $last_t = $t;
+    $last_val = $val;
+  }
+  ImageString(
+    $im,
+    5,
+    $width+$legend_line_length,
+    scale($last_val,0,$val_max,-$height+2*$border,isset($_GET["log"]))+$border - ImageFontHeight(5)/2,
+    " ".$data[$t_max],
+    $color
+  );
+};
+
+ImageRectangle($im, $legend_line_length, 0, $width-1+$legend_line_length, $height-1, $foreground_color);
+
+ImageString($im, 5, $legend_line_length, $height + 2*$legend_line_length + 2*ImageFontHeight(5), "( ".trim(shell_exec("uptime | sed 's|^.*\\s\\(load\\)|\\1|'"))." )", $foreground_color);
+
+$xpos = $legend_line_length;
+foreach ($print_columns as $column) {
+  print_graph($values[$column], $colors[$column]);
+  ImageString($im, 5, $xpos, $height + $legend_line_length + ImageFontHeight(5), substr($column,0,-strlen("_count")), $colors[$column]);
+  $xpos += (strlen($column) - strlen("_count") + 1.75) * ImageFontWidth(5);
+}
+
+ImageString($im, 5, $legend_line_length, $height + $legend_line_length, date('Y-m-d H:i', $t_min), $foreground_color);
+$s = date('Y-m-d H:i', $t_max);
+ImageString($im, 5, $width+$legend_line_length - strlen($s)*ImageFontWidth(5), $height + $legend_line_length, $s, $foreground_color);
+
+for ($t=ceil($t_min/24/60/60); $t<=floor($t_max/24/60/60); $t++)
+  ImageLine(
+    $im,
+    scale($t*24*60*60,$t_min,$t_max,$width-2*$border,false)+$border+$legend_line_length,
+    $height,
+    scale($t*24*60*60,$t_min,$t_max,$width-2*$border,false)+$border+$legend_line_length,
+    $height+$legend_line_length,
+    $foreground_color
+  );
+
+for ($val=0; $val<=$val_max;) {
+  ImageLine(
+    $im,
+    0,
+    scale($val,0,$val_max,-$height+2*$border,isset($_GET["log"]))+$border,
+    $legend_line_length,
+    scale($val,0,$val_max,-$height+2*$border,isset($_GET["log"]))+$border,
+    $foreground_color
+  );
+  if (! isset($_GET["log"]))
+    $val+=pow(10,round(log($val_max)/log(10))-1);
+  elseif ($val==0)
+    $val++;
+  else
+    $val=$val*10;
+}
+
+// ImageString ($im, 1, 5, 5, "Test-String ".rand(), $foreground_color);
+
+header ("Content-type: image/png");
+
+ImagePNG ($im);
+
+?>
diff --git a/scripts/status.php b/scripts/status.php
new file mode 100644
index 0000000..5050b4b
--- /dev/null
+++ b/scripts/status.php
@@ -0,0 +1,30 @@
+<html>
+<head>
+<?php
+
+print "<title>Build master status</title>\n";
+print "<link rel=\"stylesheet\" type=\"text/css\" href=\"/static/style.css\">\n";
+print "</head>\n";
+print "<body>\n";
+print "<a href=\"/\">Start page</a><br>\n";
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+
+if ( ! $result = $mysql -> query(
+    "SELECT MAX(`package_sources`.`commit_time`) AS `last`" .
+    "FROM `package_sources`"
+  ))
+  die($mysql->error);
+
+if ($result -> num_rows > 0) {
+
+  $row = $result->fetch_assoc();
+  print "latest package source is from " . $row["last"] . ".<br>\n";
+}
+
+?>
+</body>
+</html>
diff --git a/scripts/to-delete.php b/scripts/to-delete.php
new file mode 100644
index 0000000..dfa5a39
--- /dev/null
+++ b/scripts/to-delete.php
@@ -0,0 +1,61 @@
+<html>
+<head>
+<title>List of packages to be deleted</title>
+<link rel="stylesheet" type="text/css" href="/static/style.css">
+</head>
+<body>
+<?php
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+
+$result = $mysql -> query(
+  "SELECT " .
+  "`repositories`.`name` AS `repo`," .
+  "`binary_packages`.`pkgname`," .
+  "`binary_packages`.`epoch`," .
+  "`binary_packages`.`pkgver`," .
+  "`binary_packages`.`pkgrel`," .
+  "`binary_packages`.`sub_pkgrel`," .
+  "`architectures`.`name` AS `arch` " .
+  "FROM `binary_packages` " .
+  "JOIN `architectures` ON `binary_packages`.`architecture`=`architectures`.`id` " .
+  "JOIN `repositories` ON `binary_packages`.`repository`=`repositories`.`id` " .
+  "WHERE `binary_packages`.`is_to_be_deleted` " .
+  "AND NOT `repositories`.`name` IN (\"build-support\",\"build-list\",\"deletion-list\")"
+);
+if ($result -> num_rows > 0) {
+
+  $count = 0;
+
+  while ($row = $result->fetch_assoc()) {
+    $rows[$count] =
+      $row["repo"] . "/" .
+      $row["pkgname"] . "-";
+    if ($row["epoch"] != "0")
+      $rows[$count] =
+        $rows[$count] .
+        $row["epoch"] . ":";
+    $rows[$count] =
+      $rows[$count] .
+      $row["pkgver"] . "-" .
+      $row["pkgrel"] . "." .
+      $row["sub_pkgrel"] . "-" .
+      $row["arch"] . ".pkg.tar.xz";
+    $count++;
+  }
+
+  sort($rows);
+
+  foreach ($rows as $row) {
+    print $row."<br>\n";
+  }
+} else {
+  print "No packages are to be deleted.\n";
+}
+
+?>
+</body>
+</html>
diff --git a/scripts/todos.php b/scripts/todos.php
new file mode 100644
index 0000000..99e991c
--- /dev/null
+++ b/scripts/todos.php
@@ -0,0 +1,95 @@
+<?php
+
+$mysql = new mysqli("localhost", "webserver", "empty", "buildmaster");
+if ($mysql->connect_error) {
+  die("Connection failed: " . $mysql->connect_error);
+}
+
+$result = $mysql -> query(
+  "SELECT DISTINCT " .
+  "`todos`.`id`," .
+  "`todos`.`file`," .
+  "`todos`.`line`," .
+  "`todos`.`description` " .
+  "FROM `todos`;"
+);
+
+if (isset($_GET["graph"])) {
+
+  if ($result -> num_rows > 0) {
+
+    while ($row = $result->fetch_assoc())
+      $knot_rows[$row["id"]] =
+        $row["file"]. " (line ".$row["line"]."):\\n".str_replace("\"","\\\"",$row["description"]);
+
+    unset($knots);
+    foreach ($knot_rows as $knot)
+      $knots=$knots . "\"" . $knot . "\";\n";
+
+  }
+
+  $result = $mysql -> query(
+    "SELECT DISTINCT " .
+    "`todo_links`.`dependent`," .
+    "`todo_links`.`depending_on` " .
+    "FROM `todo_links`;"
+  );
+
+  if ($result -> num_rows > 0) {
+    $count = 0;
+    while ($row = $result->fetch_assoc()) {
+      $link_rows[$count]["dependent"] =
+        $knot_rows[$row["dependent"]];
+      $link_rows[$count]["depending_on"] =
+        $knot_rows[$row["depending_on"]];
+      $count++;
+    }
+
+    unset($edges);
+    foreach ($link_rows as $link)
+      $edges=$edges . "\"" . $link["depending_on"] . "\" -> \"" . $link["dependent"] . "\";\n";
+  }
+
+  $knots = str_replace("\$","\\\$",$knots);
+  $edges = str_replace("\$","\\\$",$edges);
+
+  header ("Content-type: image/png");
+  passthru(
+    "dot -Tpng -o/dev/stdout /dev/stdin <<EOF\n" .
+    "digraph dependencies {\n" .
+    "rankdir=LR;\n" .
+    "fontname=dejavu;\n" .
+    $knots .
+    $edges .
+    "}\n" .
+    "EOF\n"
+  );
+
+} else { // isset($_GET["graph"])
+
+  if ($result -> num_rows > 0) {
+
+    print "<html>\n";
+    print "<head>\n";
+    print "<title>Todos in the build scripts</title>\n";
+    print "</head>\n";
+    print "<body>\n";
+
+    while ($row = $result->fetch_assoc()) {
+      print "<a href=\"#TODO" . $row["id"] . "\">TODO #" . $row["id"] . "</a>";
+      print " - ";
+      print "<a href=\"https://github.com/archlinux32/builder/blob/master/" . $row["file"] . "#L" . $row["line"] . "\">" . $row["file"] . "(line " . $row["line"] . ")</a>";
+      print ":<br>\n";
+      print str_replace("\\n","<br>\n",$row["description"]);
+      print "<br>\n";
+      print "<br>\n";
+    }
+
+    print "</body>\n";
+    print "</html>\n";
+
+  }
+
+}
+
+?>
author	Erich Eckner <git@eckner.net>	2018-04-11 16:54:13 +0200
committer	Erich Eckner <git@eckner.net>	2018-04-11 16:54:13 +0200
commit	6d93913ca2893a7fc5499c0cb5d6e70831e7af50 (patch)
tree	2e3b2e0a9779f75200648857852a9bc8171750ef
download	archweb32-6d93913ca2893a7fc5499c0cb5d6e70831e7af50.tar.xz