slabpkg/slabpkg.c

// Simple package manager.
// Only needs stdint.h, stdlib.h, stdio.h, string.h & mkdir()
// May be extended with optional extras (checking timestamps etc.) later.
// This is NEW CODE written by Zak Yani Star Fenton, public domain (UNLICENSE license)

#ifndef PKG_MKDIR_SIMPLE
#define PKG_MKDIR_LINUX
#endif

// For reliable integer sizing
#include <stdint.h>
// For malloc, free
#include <stdlib.h>
// For FILE, fopen, fread, fwrite, fclose, printf, fprintf, stdout, stderr
#include <stdio.h>
// For strcmp, strdup
#include <string.h>
// For mkdir
#ifdef PKG_MKDIR_LINUX
#include <sys/stat.h>
#else
#ifdef PKG_MKDIR_SIMPLE
int mkdir(const char* dirname);
#else
#error Either (-D) PKG_MKDIR_LINUX or PKG_MKDIR_SIMPLE must be defined
#endif
#endif

// The compressed buffer needs to be a bit over twice size the regular
// buffer to account for worst case scenario.
#define PKG_BUFFERSIZE    4096
#define PKG_COMPRBUFSIZE  8224 

#define PKG_JOBTYPE_BUILD       1
#define PKG_JOBTYPE_INSTALL     2
#define PKG_JOBTYPE_REMOVE      3
#define PKG_JOBTYPE_SUM         4
#define PKG_JOBTYPE_COMPRESS    5
#define PKG_JOBTYPE_DECOMPRESS  6

#define PKG_FLAGS_DIRECTORY     0x100
#define PKG_FLAGS_EXECUTABLE    0x200
#define PKG_FLAGS_PROVIDES      0x300
#define PKG_FLAGS_REQUIRES      0x400
#define PKG_FLAGS_SUGGESTS      0x500
#define PKG_FLAGS_CONFLICTS     0x600

typedef struct pkg_job pkg_job_t;
typedef struct pkg_project pkg_project_t;
typedef struct pkg_sum pkg_sum_t;
typedef struct pkg_archive pkg_archive_t;
typedef struct pkg_version pkg_version_t;
typedef struct pkg_versionmeta pkg_versionmeta_t;

typedef long long pkg_size_t;
typedef unsigned long long pkg_usize_t;

struct pkg_job {
  char** names;
  char* target;
  char* installroot;
  char* dbroot;
  int usedb;
  int nnames;
  int jobtype;
  int verbose;
};

struct pkg_project {
  char* projdir;
  char* projfile;
};

struct pkg_sum {
  char* name;
  pkg_sum_t* next;
  pkg_usize_t offset;
  pkg_usize_t cmprsize; // Compressed/in-package size
  pkg_usize_t extrsize; // Extracted/real-file size
  pkg_usize_t checksum64;
  unsigned int flags; // This field is set to zero normally
  unsigned int checksum32;
  unsigned int nameoffset;
  unsigned int namelength;
  unsigned int namechecksum;
};

struct pkg_archive {
  int version;
  int nheaders;
  pkg_sum_t* headerlist;
};

struct pkg_versionmeta {
  const char* value;
  pkg_versionmeta_t* next;
};

struct pkg_version {
  int major;
  int minor;
  int patch;
  pkg_versionmeta_t* prerelease;
  pkg_versionmeta_t* meta;
};

// Buffer used for most reads/writes, could be dynamically allocated later.
char pkg_linebuffer[PKG_BUFFERSIZE];
char pkg_comprbuffer[PKG_COMPRBUFSIZE];

int pkg_mkdir(const char* path) {
#ifdef PKG_MKDIR_LINUX
  return mkdir(path, 0775);
#endif
#ifdef PKG_MKDIR_SIMPLE
  return mkdir(path);
#endif
}

int pkg_remove(const char* path, int isdir) {
  return unlink(path);
}

int pkg_versionmeta_alldigits(pkg_versionmeta_t* m) {
  const char* s = m->value;
  while (*s) {
    if (*s < '0' || *s > '9') {
      return 0;
    }
    s++;
  }
  return 1;
}

int pkg_versionmeta_compare(pkg_versionmeta_t* left, pkg_versionmeta_t* right) {
  int ldigits = pkg_versionmeta_alldigits(left);
  int rdigits = pkg_versionmeta_alldigits(right);
  
  if (ldigits && !rdigits) {
    return -1;
  } else if (rdigits && !ldigits) {
    return 1;
  } else if (!rdigits && !ldigits) {
    return strcmp(left->value, right->value);
  } else {
    const char* l = left->value;
    const char* r = right->value;
    while (*l == '0') {
      l++;
    }
    while (*r == '0') {
      r++;
    }
    size_t llen = strlen(l);
    size_t rlen = strlen(r);
    if (llen < rlen) {
      return -1;
    } else if (llen > rlen) {
      return 1;
    } else {
      return strcmp(l, r); // ASCII comparison will do for numbers too if same number of digits
    }
  }
}

// Compares versions, returns 0 if equal, -1 if left < right or 1 if
// left > right. Treats NULLs as lower-than any real version.
int pkg_version_compare(pkg_version_t* left, pkg_version_t* right) {
  // A NULL is just treated as less-than any real version.
  if (left == NULL && right == NULL) {
    return 0;
  } else if (left == NULL) {
    return -1;
  } else if (right == NULL) {
    return 1;
  }
  
  if (left->major < right->major) {
    return -1;
  } else if (left->major > right->major) {
    return 1;
  }
  
  if (left->minor < right->minor) {
    return -1;
  } else if (left->minor > right->minor) {
    return 1;
  }
  
  if (left->patch < right->patch) {
    return -1;
  } else if (left->patch > right->patch) {
    return 1;
  }
  
  if (left->prerelease != NULL && right->prerelease == NULL) {
    return -1;
  } else if (left->prerelease == NULL && right->prerelease != NULL) {
    return 1;
  } else if (left->prerelease == NULL && right->prerelease == NULL) {
    return 0;
  }
  
  pkg_versionmeta_t* lpre = left->prerelease;
  pkg_versionmeta_t* rpre = right->prerelease;
  
  while (lpre != NULL && rpre != NULL) {
    lpre = lpre->next;
    rpre = rpre->next;
  }
  
  if (rpre != NULL) {
    return -1;
  } else if (lpre != NULL) {
    return 1;
  } else {
    return 0;
  }
}

int pkg_version_parseint(int* resultvar, const char* string) {
  int i = 0;
  int result = 0;
  
  while (string[i] >= '0' && string[i] <= '9') {
    result *= 10;
    result += (string[i] - '0');
    i++;
  }
  
  *resultvar = result;
  
  return i;
}

int pkg_version_isalnum(char foo) {
  if (foo >= '0' && foo <= '9') {
    return 1;
  } else if (foo >= 'a' && foo <= 'z') {
    return 1;
  } else if (foo >= 'A' && foo <= 'Z') {
    return 1;
  } else if (foo == '-') {
    // Hyphens are treated as alnum in version meta
    return 1;
  } else {
    return 0;
  }
}

int pkg_versionmeta_parse1(pkg_versionmeta_t** resultvar, const char* string) {
  int i = 0;
  
  while (pkg_version_isalnum(string[i])) {
    i++;
  }
  
  if (i > 0) {
    pkg_versionmeta_t* newmeta = malloc(sizeof(pkg_versionmeta_t));
    if (newmeta == NULL) {
      return -1;
    }
    newmeta->value = strndup(string, i);
    newmeta->next = NULL;
    *resultvar = newmeta;
  }
  
  return i;
}

int pkg_versionmeta_parse(pkg_versionmeta_t** resultvar, const char* string) {
  int i = 0;
  
  pkg_versionmeta_t* firstmeta = NULL;
  pkg_versionmeta_t* oldmeta = NULL;
  do {
    pkg_versionmeta_t* newmeta;
    int tmp = pkg_versionmeta_parse1(&newmeta, string+i);
    if (tmp < 1) {
      return -1;
    }
    i += tmp;
    if (oldmeta == NULL) {
      firstmeta = newmeta;
    } else {
      oldmeta->next = newmeta;
    }
    oldmeta = newmeta;
  } while (string[i++] == '.');
  
  *resultvar = firstmeta;
  
  return i;
}

// Parses a version string, returns a positive integer indicating the
// number of chars parsed on success and <1 if no version string was
// detected.
int pkg_version_parse(pkg_version_t* version, const char* string) {
  int i = 0;
  
  while (string[i] == ' ') {
    i++;
  }
  
  int tmp = pkg_version_parseint(&version->major, string+i);
  if (tmp < 1) {
    return -1;
  }
  i += tmp;
  if (string[i] != '.') {
    return -1;
  }
  i++;
  tmp = pkg_version_parseint(&version->minor, string+i);
  if (tmp < 1) {
    return -1;
  }
  i += tmp;
  if (string[i] != '.') {
    return -1;
  }
  i++;
  tmp = pkg_version_parseint(&version->patch, string);
  if (tmp < 1) {
    return -1;
  }
  i += tmp;
  
  if (string[i] == '-') {
    i++;
    tmp = pkg_versionmeta_parse(&version->prerelease, string+i);
    if (tmp < 1) {
      return -1;
    }
    i += tmp;
  } else {
    version->prerelease = NULL;
  }
  
  if (string[i] == '+') {
    i++;
    tmp = pkg_versionmeta_parse(&version->meta, string+i);
    if (tmp < 1) {
      return -1;
    }
    i += tmp;
  } else {
    version->meta = NULL;
  }
  
  return i;
}

size_t pkg_firstlongestmatch(unsigned char* data, size_t dlength, unsigned char* key, size_t klength, size_t* indexvar) {
  if (klength < 1) {
    *indexvar = -1;
    return 0;
  }
  size_t mlength = 0;
  size_t i = 0;
  while (i < dlength) {
    size_t j = 0;
    while (j < klength && i + j < dlength && data[i + j] == key[j]) {
      j++;
    }
    if (j > mlength) {
      mlength = j;
      *indexvar = i;
      if (mlength == klength) {
        return mlength;
      }
    }
    i += j ? j : 1;
  }
  return mlength;
}

#define PKG_COMPRESS_OUTBYTE(b) if (outi < olength) {output[outi] = (unsigned char) (b);} outi++
#define PKG_COMPRESS_FLUSHUNCOMPRESSED() if (unflushed) { \
  PKG_COMPRESS_OUTBYTE((unsigned char) unflushed); \
  for (size_t flushi = 0; flushi < unflushed; flushi++) { \
    PKG_COMPRESS_OUTBYTE(input[(i - unflushed) + flushi]); \
  } \
  unflushed = 0; \
}

size_t pkg_compressv1(unsigned char* input, size_t ilength, unsigned char* output, size_t olength) {
  if (ilength > 64*1024) {
    return -1;
  }
  size_t i = 0;
  size_t outi = 0;
  size_t unflushed = 0;
  while (i < ilength) {
    size_t mindex = -1;
    size_t mlength = pkg_firstlongestmatch(input, i, input + i, ilength - i > 32 ? 32 : ilength - i, &mindex);
    /*if (mlength > 2 && mindex < 256) {
      PKG_COMPRESS_FLUSHUNCOMPRESSED();
      PKG_COMPRESS_OUTBYTE(0 - (mlength + 32));
      PKG_COMPRESS_OUTBYTE(mindex);
      i += mlength;
    } else*/ if (mlength > 3) {
      PKG_COMPRESS_FLUSHUNCOMPRESSED();
      PKG_COMPRESS_OUTBYTE(0 - mlength);
      PKG_COMPRESS_OUTBYTE(mindex);
      PKG_COMPRESS_OUTBYTE(mindex >> 8);
      i += mlength;
    } else { // TODO: This will need to be changed to write multiple uncompressed bytes!
      if (unflushed >= 100) {
        PKG_COMPRESS_FLUSHUNCOMPRESSED();
      }
      //PKG_COMPRESS_OUTBYTE(1);
      //PKG_COMPRESS_OUTBYTE(input[i]);
      /*if (unflushed == 0 && input[i] >= 'a' && input[i] <= 'z') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + (input[i] - 'a')));
      } else if (unflushed == 0 && input[i] >= 'A' && input[i] <= 'Z') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26 + (input[i] - 'A')));
      } else if (unflushed == 0 && input[i] == 0) {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 0));
      } else if (unflushed == 0 && input[i] == 1) {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 1));
      } else if (unflushed == 0 && input[i] == 2) {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 2));
      } else if (unflushed == 0 && input[i] == 0xFF) {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 3));
      } else if (unflushed == 0 && input[i] == ' ') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 4));
      } else if (unflushed == 0 && input[i] == '.') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 5));
      } else if (unflushed == 0 && input[i] == ',') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 6));
      } else if (unflushed == 0 && input[i] == '(') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 7));
      } else if (unflushed == 0 && input[i] == ')') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 8));
      } else if (unflushed == 0 && input[i] == '\n') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 9));
      } else if (unflushed == 0 && input[i] == '/') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 10));
      } else if (unflushed == 0 && input[i] == ';') {
        PKG_COMPRESS_OUTBYTE(0 - (64 + 26*2 + 11));
      } else {*/
        unflushed++;
      /*}*/
      i++;
    }
  }
  PKG_COMPRESS_FLUSHUNCOMPRESSED();
  return outi;
}

size_t pkg_decompressv1(unsigned char* input, size_t ilength, unsigned char* output, size_t olength) {
  if (ilength > 129*1024) {
    return -1;
  }
  size_t i = 0;
  size_t outi = 0;
  while (i < ilength) {
    char b = (char) input[i];
    if (b < 0) {
      if (i+2 < ilength) {
        size_t len = 0 - b;
        size_t idx = (((size_t)input[i+1]) & 0xFF) | ((((size_t)input[i+2]) & 0xFF) << 8);
        i += 3;
        if (idx+len > outi) {
          return -1;
        }
        memcpy(output + outi, output + idx, len);
        outi += len;
      } else {
        return -1;
      }
    } else {
      if (i + b >= ilength) {
        return -1;
      }
      memcpy(output + outi, input + i + 1, b);
      i += 1 + b;
      outi += b;
    }
  }
  return outi;
}

char* pkg_ezstrcat(const char* a, const char* b) {
  int la = strlen(a);
  int lb = strlen(b);
  char* output = malloc(la + lb + 1);
  if (output == NULL) {
    fprintf(stderr, "MEMORY ERROR: Failed to allocate memory for string concatenation!\n");
    return NULL;
  }
  int i;
  for (i = 0; i < la; i++) {
    output[i] = a[i];
  }
  for (; i < la + lb; i++) {
    output[i] = b[i - la];
  }
  output[i] = 0;
  return output;
}

uint32_t pkg_cstringhash32(const char* cstring) {
  uint32_t result = 0x811C9DC5U;
  uint32_t chud = 0x01000193U;
  while (*cstring) {
    result ^= ((uint32_t) *cstring++) & 0xFFU;
    result *= chud;
  }
  return result;
}

int pkg_sum_init(pkg_sum_t* sum, char* filename, int dupfilename) {
  if (sum == NULL) {
    return -1;
  }
  sum->cmprsize = 0;
  sum->extrsize = 0;
  sum->checksum64 = 0x811C9DC5BADC0DE5ULL;
  sum->checksum32 = 0x811C9DC5U;
  sum->next = NULL;
  if (dupfilename) {
    sum->name = strdup(filename);
    if (sum->name == NULL) {
      return -1;
    }
  } else {
    sum->name = filename;
  }
  return 0;
}

int pkg_sum_append(pkg_sum_t* sum, void* data, pkg_usize_t size) {
  unsigned char* bytes = data;
  if (sum == NULL) {
    return -1;
  }
  if (size == 0) {
    return 0;
  }
  if (data == NULL) {
    return -1;
  }
  pkg_usize_t chud64 = 0x0100019301000193ULL;
  unsigned int chud32 = 0x01000193U;
  for (pkg_usize_t i = 0; i < size; i++) {
    sum->checksum64 ^= ((pkg_usize_t) bytes[i]) & 0xFFULL;
    sum->checksum64 *= chud64;
    sum->checksum32 ^= ((unsigned int) bytes[i]) & 0xFFU;
    sum->checksum32 *= chud32;
  }
  sum->extrsize += size;
  return 0;
}

pkg_sum_t* pkg_archive_addheader(pkg_archive_t* archive, const char* name) {
  if (archive == NULL) {
    return NULL;
  }
  pkg_sum_t* newheader = malloc(sizeof(pkg_sum_t));
  if (newheader == NULL) {
    return NULL;
  }
  if (pkg_sum_init(newheader, name, 1) != 0) {
    free(newheader);
    return NULL;
  }
  pkg_sum_t* lastheader = archive->headerlist;
  while (lastheader != NULL && lastheader->next != NULL) {
    lastheader = lastheader->next;
  }
  if (lastheader == NULL) {
    archive->headerlist = newheader;
  } else {
    lastheader->next = newheader;
  }
  return newheader;
}

int pkg_archive_init(pkg_archive_t* archive) {
  if (archive == NULL) {
    return -1;
  }
  archive->version = 0;
  archive->headerlist = NULL;
  return 0;
}

size_t pkg_writefzeros(size_t nzeros, FILE* f) {
  char zero = 0;
  size_t nwritten = 0;
  while (nwritten < nzeros && fwrite(&zero, 1, 1, f) == 1) {
    nwritten++;
  }
  return nwritten;
}

int pkg_archive_writef32(pkg_archive_t* archive, unsigned int value, FILE* output, pkg_sum_t* sum) {
  unsigned char bytes[4];
  bytes[0] = (unsigned char) value;
  bytes[1] = (unsigned char) (value >> 8);
  bytes[2] = (unsigned char) (value >> 16);
  bytes[3] = (unsigned char) (value >> 24);
  int nwritten = (int) fwrite(bytes, 1, 4, output);
  if (sum != NULL) {
    pkg_sum_append(sum, bytes, nwritten);
  }
  return nwritten;
}

int pkg_archive_writef64(pkg_archive_t* archive, pkg_usize_t value, FILE* output, pkg_sum_t* sum) {
  int nwritten = pkg_archive_writef32(archive, (unsigned int) value, output, sum);
  if (nwritten != 4) {
    return nwritten;
  }
  nwritten += pkg_archive_writef32(archive, (unsigned int) (value>>32), output, sum);
  return nwritten;
}

int pkg_archive_readf32(pkg_archive_t* archive, unsigned int* valuevar, FILE* input, pkg_sum_t* sum) {
  unsigned char bytes[4];
  int nread = (int) fread(bytes, 1, 4, input);
  if (nread != 4) {
    fprintf(stderr, "WARNING: fread of 4 bytes read %d bytes!\n", nread);
  }
  *valuevar = bytes[0] | ((unsigned int) bytes[1])<<8 | ((unsigned int) bytes[2])<<16 | ((unsigned int) bytes[3])<<24;
  if (sum != NULL) {
    pkg_sum_append(sum, bytes, nread);
  }
  return nread;
}

int pkg_archive_readf64(pkg_archive_t* archive, pkg_usize_t* valuevar, FILE* input, pkg_sum_t* sum) {
  unsigned int valuelow, valuehigh;
  int nread = pkg_archive_readf32(archive, &valuelow, input, sum);
  if (nread != 4) {
    return nread;
  }
  nread += pkg_archive_readf32(archive, &valuehigh, input, sum);
  *valuevar = ((pkg_usize_t) valuelow) | (((pkg_usize_t) valuehigh) << 32);
  return nread;
}

int pkg_archive_readheaders(pkg_archive_t* archive, FILE* input) {
  char buffer[9];
  buffer[8] = 0;
  if (archive == NULL) {
    fprintf(stderr, "INTERNAL ERROR: Invalid archive structure!\n");
    return -1;
  }
  pkg_usize_t nread = 0;
  if (fread(buffer, 1, 8, input) != 8) {
    fprintf(stderr, "WRITE ERROR: Reading from archive file failed!\n");
    return -1;
  }
  nread += 8;
  if (strcmp(buffer, "DEMOPKG0")) {
    fprintf(stderr, "BAD PACKAGE DATA: Magic number doesn't match!\n");
    return -1;
  }
  if (fread(buffer, 1, 8, input) != 8) {
    fprintf(stderr, "WRITE ERROR: Reading from archive file failed!\n");
    return -1;
  }
  nread += 8;
  fprintf(stderr, "Got reserved string '%s'\n", buffer);
  
  unsigned int nfiles, headersize;
  
  nread += pkg_archive_readf32(archive, &nfiles, input, NULL); // Number of header entries
  nread += pkg_archive_readf32(archive, &headersize, input, NULL); // Size of per-entry header
  
  if (headersize != 48) {
    fprintf(stderr, "BAD PACKAGE DATA: Expected a header size of 48 bytes per encoded file but got %d\n", headersize);
    return -1;
  }
  
  fprintf(stderr, "Loading %d file headers...\n", nfiles);
  
  pkg_sum_t* filehdr;
  
  for (unsigned int i = 0; i < nfiles; i++) {
    filehdr = pkg_archive_addheader(archive, "<LOADING...>");
    if (filehdr == NULL) {
      fprintf(stderr, "INTERNAL ERROR: Failed to add new header to archive struct!\n");
      return -1;
    }
    
    pkg_sum_t filehdrsum;
    
    if (pkg_sum_init(&filehdrsum, "<TMP-HDR>", 0) != 0) {
      fprintf(stderr, "INTERNAL ERROR: pkg_sum_init failed!\n");
      return -1;
    }
    
    size_t nread2 = 0;
    unsigned int hdrsum;
    nread2 += pkg_archive_readf64(archive, &filehdr->offset, input, &filehdrsum);
    
    nread2 += pkg_archive_readf64(archive, &filehdr->cmprsize, input, &filehdrsum);
    
    nread2 += pkg_archive_readf64(archive, &filehdr->extrsize, input, &filehdrsum);
    
    nread2 += pkg_archive_readf32(archive, &filehdr->flags, input, &filehdrsum);
    nread2 += pkg_archive_readf32(archive, &filehdr->checksum32, input, &filehdrsum);
    
    nread2 += pkg_archive_readf32(archive, &filehdr->nameoffset, input, &filehdrsum);
    nread2 += pkg_archive_readf32(archive, &filehdr->namelength, input, &filehdrsum);
    
    nread2 += pkg_archive_readf32(archive, &filehdr->namechecksum, input, &filehdrsum);
    nread2 += pkg_archive_readf32(archive, &hdrsum, input, NULL);
    
    if (hdrsum != filehdrsum.checksum32) {
      fprintf(stderr, "VERIFICATION ERROR: Header checksum for file #%d in archive doesn't match. Recorded checksum 0x%X calculated checksum 0x%X!\n", i, filehdrsum.checksum32, hdrsum);
      return -1;
    }
    //fprintf(stderr, "For file header #%d: recorded checksum 0x%X calculated checksum 0x%X!\n", i, filehdrsum.checksum32, hdrsum);
    
    if (nread2 != 48) {
      fprintf(stderr, "READ ERROR: Expected to read 48 bytes of file header info but read %d instead!\n", (int) nread2);
      return -1;
    }
    
    nread += nread2;
  }
  
  filehdr = archive->headerlist;
  size_t nreadstringdata;
  int filenum = 0;
  while (filehdr != NULL) {
    size_t n = fread(pkg_linebuffer, 1, filehdr->namelength, input);
    if (n != filehdr->namelength) {
      fprintf(stderr, "READ ERROR: Failed while reading filename strings!\n");
      return -1;
    }
    filehdr->name = strndup(pkg_linebuffer, filehdr->namelength);
    unsigned int namesum = pkg_cstringhash32(filehdr->name);
    if (namesum != filehdr->namechecksum) {
      fprintf(stderr, "VERIFICATION ERROR: Filename checksum for file #%d in archive doesn't match. Recorded checksum 0x%X calculated checksum 0x%X!\n", filenum, filehdr->namechecksum, namesum);
      return -1;
    }
    //fprintf(stderr, "For file header #%d in archive: Recorded checksum 0x%X calculated checksum 0x%X!\n", filenum, filehdr->namechecksum, namesum);
    fprintf(stderr, "Loaded header for file #%d in archive: extracted size %lu, compressed size %lu, data checksum 0x%X, name '%s'\n", filenum, (long) filehdr->extrsize, filehdr->cmprsize, filehdr->checksum32, filehdr->name);
    filehdr = filehdr->next;
    filenum++;
  }
  
  return 0;
}

int pkg_archive_writeheaders(pkg_archive_t* archive, FILE* output, int blankrun) {
  if (archive == NULL) {
    fprintf(stderr, "INTERNAL ERROR: Invalid archive structure!\n");
    return -1;
  }
  pkg_usize_t nwritten = 0;
  if (fwrite((blankrun ? "UNFINSHD" : "DEMOPKG0"), 1, 8, output) != 8) {
    fprintf(stderr, "WRITE ERROR: Writing to archive file failed!\n");
    return -1;
  }
  nwritten += 8;
  if (fwrite((blankrun ? "PARTWRIT" : "RESERVED"), 1, 8, output) != 8) {
    fprintf(stderr, "WRITE ERROR: Writing to archive file failed!\n");
    return -1;
  }
  nwritten += 8;
  
  unsigned int nfiles = 0;
  pkg_sum_t* filehdr = archive->headerlist;
  while (filehdr != NULL) {
    filehdr = filehdr->next;
    nfiles++;
  }
  
  nwritten += pkg_archive_writef32(archive, nfiles, output, NULL); // Number of header entries
  nwritten += pkg_archive_writef32(archive, 48, output, NULL); // Size of per-entry header
  
  filehdr = archive->headerlist;
  unsigned int nameoffset = 0;
  while (filehdr != NULL) {
    unsigned int namelen = (unsigned int) strlen(filehdr->name);
    
    pkg_sum_t filehdrsum;
    
    if (pkg_sum_init(&filehdrsum, "<TMP-HDR>", 0) != 0) {
      fprintf(stderr, "INTERNAL ERROR: pkg_sum_init failed!\n");
      return -1;
    }
    
    size_t nwritten2 = 0;
    nwritten2 += pkg_archive_writef64(archive, filehdr->offset, output, &filehdrsum);
    
    nwritten2 += pkg_archive_writef64(archive, filehdr->cmprsize, output, &filehdrsum);
    
    nwritten2 += pkg_archive_writef64(archive, filehdr->extrsize, output, &filehdrsum);
    
    nwritten2 += pkg_archive_writef32(archive, filehdr->flags, output, &filehdrsum);
    nwritten2 += pkg_archive_writef32(archive, filehdr->checksum32, output, &filehdrsum);
    
    nwritten2 += pkg_archive_writef32(archive, nameoffset, output, &filehdrsum);
    nwritten2 += pkg_archive_writef32(archive, namelen, output, &filehdrsum);
    
    nwritten2 += pkg_archive_writef32(archive, pkg_cstringhash32(filehdr->name), output, &filehdrsum);
    nwritten2 += pkg_archive_writef32(archive, filehdrsum.checksum32, output, NULL);
    
    if (nwritten2 != 48) {
      fprintf(stderr, "WRITE ERROR: Expected to write 48 bytes of file header info but wrote %d instead!\n", (int) nwritten2);
      return -1;
    }
    
    nwritten += nwritten2;
    
    nameoffset += namelen;
    
    filehdr = filehdr->next;
  }
  
  filehdr = archive->headerlist;
  while (filehdr != NULL) {
    size_t nwritten2 = 0;
    nwritten2 = fwrite(filehdr->name, 1, strlen(filehdr->name), output);
    if (nwritten2 != strlen(filehdr->name)) {
      fprintf(stderr, "WRITE ERROR: Failed while writing filename strings!\n");
      return -1;
    }
    
    nwritten += nwritten2;
    
    filehdr = filehdr->next;
  }
  
  fprintf(stderr, "Written %ld bytes of header\n", (long) nwritten);
  
  return 0;
}

int pkg_archive_extractcontents(pkg_archive_t* archive, FILE* input, const char* installroot) {
  if (archive == NULL || archive->headerlist == NULL) {
    fprintf(stderr, "INTERNAL ERROR: Invalid or uninitialised archive structure!\n");
    return -1;
  }
  pkg_usize_t nwritten = 0;
  pkg_sum_t* filehdr = archive->headerlist;
  while (filehdr != NULL) {
    char* outputname = pkg_ezstrcat(installroot, filehdr->name); // Note, this assumes one or the other includes a path separator
    
    if (filehdr->flags & PKG_FLAGS_DIRECTORY) {
      fprintf(stderr, "Creating directory '%s'...\n", outputname);
      int mkdirresult = pkg_mkdir(outputname);
      fprintf(stderr, "mkdir of '%s' returned %d\n", outputname, mkdirresult);
    } else {
      fprintf(stderr, "Creating file '%s'...\n", outputname);
      FILE* f = fopen(outputname, "w");
      if (f == 0) {
        fprintf(stderr, "BAD FILE: Failed to open '%s' for writing!\n", outputname);
        return -1;
      }
      size_t nreadnow, nwrittennow;
      size_t remaining = filehdr->extrsize;
      pkg_sum_t datasum;
      if (pkg_sum_init(&datasum, "<TMP-FILEDATA>", 0) != 0) {
        fprintf(stderr, "INTERNAL ERROR: pkg_sum_init failed!\n");
        return -1;
      }
      do {
        size_t part = PKG_BUFFERSIZE > remaining ? remaining : PKG_BUFFERSIZE;
        nreadnow = fread(pkg_linebuffer, 1, part, input);
        if (nreadnow > 0) {
          nwrittennow = fwrite(pkg_linebuffer, 1, nreadnow, f);
          if (pkg_sum_append(&datasum, pkg_linebuffer, nreadnow) != 0) {
            fprintf(stderr, "INTERNAL ERROR: pkg_sum_append failed!\n");
            free(outputname);
            fclose(f);
            return -1;
          }
          if (nwrittennow != nreadnow) {
            fprintf(stderr, "WRITE ERROR: Error while extracting contents of '%s' from the archive!\n", outputname);
            free(outputname);
            fclose(f);
            return -1;
          }
          nwritten += nwrittennow;
          remaining -= nwrittennow;
        }
      } while(remaining > 0);
      
      fclose(f);
      
      if (datasum.checksum32 != filehdr->checksum32) {
        fprintf(stderr, "VERIFICATION ERROR: File data checksums for '%s' don't match. Recorded checksum 0x%X, calculated checksum 0x%X!\n", filehdr->name, filehdr->checksum32, datasum.checksum32);
        return -1;
      }
    }
    
    filehdr = filehdr->next;
  }
  return 0;
}

int pkg_archive_writecontents(pkg_archive_t* archive, FILE* output, const char* inputdir) {
  if (archive == NULL || archive->headerlist == NULL) {
    fprintf(stderr, "INTERNAL ERROR: Invalid or uninitialised archive structure!\n");
    return -1;
  }
  pkg_usize_t nwritten = 0;
  pkg_sum_t* filehdr = archive->headerlist;
  while (filehdr != NULL) {
    if (filehdr->flags & PKG_FLAGS_DIRECTORY) {
      fprintf(stderr, "Skipping writing of file contents for '%s' (is directory)\n", filehdr->name);
    } else {
      char* inputname = pkg_ezstrcat(inputdir, filehdr->name); // Note, this assumes one or the other includes a path separator
      FILE* f = fopen(inputname, "rb");
      if (f == 0) {
        fprintf(stderr, "BAD FILE: Failed to open '%s' for reading!\n", inputname);
        return -1;
      }
      size_t nreadnow, nwrittennow;
      do {
        nreadnow = fread(pkg_linebuffer, 1, PKG_BUFFERSIZE, f);
        if (nreadnow > 0) {
          nwrittennow = fwrite(pkg_linebuffer, 1, nreadnow, output);
          if (pkg_sum_append(filehdr, pkg_linebuffer, nreadnow) != 0) {
            fprintf(stderr, "INTERNAL ERROR: pkg_sum_append failed!\n");
            free(inputname);
            fclose(f);
            return -1;
          }
          if (nwrittennow != nreadnow) {
            fprintf(stderr, "WRITE ERROR: Error while writing contents of '%s' into the archive!\n", inputname);
            free(inputname);
            fclose(f);
            return -1;
          }
          nwritten += nwrittennow;
        }
      } while(nreadnow > 0);
      
      fclose(f);
    }
    
    filehdr = filehdr->next;
  }
  return 0;
}

char* pkg_linein(FILE* f, pkg_size_t* sizevar) {
  char b;
  char* buffer = pkg_linebuffer;
  pkg_size_t i = 0;
  do {
    if (i == PKG_BUFFERSIZE - 1) {
      fprintf(stderr, "BAD FILE: Text line overflows buffer size!\n");
      if (sizevar) {
        *sizevar += i;
      }
      buffer[i] = 0;
      return NULL;
    }
    if (fread(&b, 1, 1, f) != 1) {
      if (sizevar) {
        *sizevar += i;
      }
      buffer[i] = 0;
      if (i == 0) {
        return NULL;
      }
      return buffer;
    }
    if (b == 0 || b == '\n') {
      if (sizevar) {
        *sizevar += i + 1;
      }
      buffer[i] = 0;
      return buffer;
    }
    buffer[i] = b;
    i++;
  } while (1);
}

int pkg_archive_uninstallcontents(pkg_archive_t* archive, FILE* input, const char* installroot) {
  if (archive == NULL || archive->headerlist == NULL) {
    fprintf(stderr, "INTERNAL ERROR: Invalid or uninitialised archive structure!\n");
    return -1;
  }
  pkg_usize_t nwritten = 0;
  pkg_sum_t* filehdr = archive->headerlist;
  while (filehdr != NULL) {
    char* outputname = pkg_ezstrcat(installroot, filehdr->name); // Note, this assumes one or the other includes a path separator
    
    if (filehdr->flags & PKG_FLAGS_DIRECTORY) {
      // Skip
    } else {
      fprintf(stderr, "Removing packaged file '%s'...\n", outputname);
      pkg_remove(outputname, 0);
    }
    
    filehdr = filehdr->next;
  }
  filehdr = archive->headerlist;
  while (filehdr != NULL) {
    char* outputname = pkg_ezstrcat(installroot, filehdr->name); // Note, this assumes one or the other includes a path separator
    
    if (filehdr->flags & PKG_FLAGS_DIRECTORY) {
      fprintf(stderr, "Removing packaged directory '%s'...\n", outputname);
      pkg_remove(outputname, 1);
    } else {
      // Skip (already deleted)
    }
    
    filehdr = filehdr->next;
  }
  return 0;
}

const char* pkg_build_linematch(const char* line, const char* cmd) {
  while (*cmd) {
    if (*line == 0) {
      return NULL;
    }
    if (*line != *cmd) {
      return NULL;
    }
    *line++;
    *cmd++;
  }
  return line;
}

int pkg_job_build(pkg_job_t* job, char* name) {
  int n = strlen(name);
  if (n < 5 || name[n-4] != '.' || name[n-3] != 'p' || name[n-2] != 'k' || name[n-1] != 'b') {
    fprintf(stderr, "BAD FILENAME: Package build file must end in .pkb\n");
    return -1;
  }
  int sepi = 0;
  for (int i = 0; name[i] != 0; i++) {
    if (name[i] == '/' || name[i] == '\\') {
      sepi = i+1;
    }
  }
  char* cd = "./";
  char* srcd;
  if (sepi == 0) {
    srcd = cd;
  } else {
    srcd = strndup(name, sepi);
  }
  char* distsubname = "dist"; // The directory packageable files should be kept
  char* distd = pkg_ezstrcat(srcd, "dist");
  char* outname = strdup(name);
  outname[strlen(outname)-1] = 'g'; // "*.pkb"-> "*.pkg"
  char* shortname = strndup(name + sepi, n - (sepi + 4)); // Trim directory path and filename extension
  
  printf("TODO build '%s' source dirname '%s' dist dirname '%s' output name '%s' shortname '%s'...\n", name, srcd, distd, outname, shortname);
  
  FILE* buildf = fopen(name, "rb");
  
  if (buildf == NULL) {
    fprintf(stderr, "BAD FILE: Failed to open package build file '%s' for reading!\n", name);
    return -1;
  }
  
  char* line;
  pkg_archive_t archive;
  if (pkg_archive_init(&archive) != 0) {
    fprintf(stderr, "INTERNAL ERROR: Failed to initialise archive structure!\n");
    fclose(buildf);
    return -1;
  }
  
  while ((line = pkg_linein(buildf, NULL)) != NULL) {
    fprintf(stderr, "Processing line '%s'...\n", line);
    
    const char* arg;
    if ((arg = pkg_build_linematch(line, "provides ")) != NULL) {
      pkg_sum_t* mkdirhdr = pkg_archive_addheader(&archive, arg);
      if (mkdirhdr == NULL) {
        fprintf(stderr, "INTERNAL ERROR: Failed to add archive header for '%s'!\n", line);
        fclose(buildf);
        return -1;
      }
      mkdirhdr->flags |= PKG_FLAGS_PROVIDES;
    } else if ((arg = pkg_build_linematch(line, "requires ")) != NULL) {
      pkg_sum_t* mkdirhdr = pkg_archive_addheader(&archive, arg);
      if (mkdirhdr == NULL) {
        fprintf(stderr, "INTERNAL ERROR: Failed to add archive header for '%s'!\n", line);
        fclose(buildf);
        return -1;
      }
      mkdirhdr->flags |= PKG_FLAGS_REQUIRES;
    } else if ((arg = pkg_build_linematch(line, "suggests ")) != NULL) {
      pkg_sum_t* mkdirhdr = pkg_archive_addheader(&archive, arg);
      if (mkdirhdr == NULL) {
        fprintf(stderr, "INTERNAL ERROR: Failed to add archive header for '%s'!\n", line);
        fclose(buildf);
        return -1;
      }
      mkdirhdr->flags |= PKG_FLAGS_SUGGESTS;
    } else if ((arg = pkg_build_linematch(line, "conflicts ")) != NULL) {
      pkg_sum_t* mkdirhdr = pkg_archive_addheader(&archive, arg);
      if (mkdirhdr == NULL) {
        fprintf(stderr, "INTERNAL ERROR: Failed to add archive header for '%s'!\n", line);
        fclose(buildf);
        return -1;
      }
      mkdirhdr->flags |= PKG_FLAGS_CONFLICTS;
    } else if ((arg = pkg_build_linematch(line, "directory ")) != NULL) {
      pkg_sum_t* mkdirhdr = pkg_archive_addheader(&archive, arg);
      if (mkdirhdr == NULL) {
        fprintf(stderr, "INTERNAL ERROR: Failed to add archive header for '%s'!\n", line);
        fclose(buildf);
        return -1;
      }
      mkdirhdr->flags |= PKG_FLAGS_DIRECTORY;
    } else if ((arg = pkg_build_linematch(line, "executable ")) != NULL) {
      pkg_sum_t* mkdirhdr = pkg_archive_addheader(&archive, arg);
      if (mkdirhdr == NULL) {
        fprintf(stderr, "INTERNAL ERROR: Failed to add archive header for '%s'!\n", line);
        fclose(buildf);
        return -1;
      }
      mkdirhdr->flags |= PKG_FLAGS_EXECUTABLE;
    } else if (pkg_archive_addheader(&archive, line) == NULL) {
      fprintf(stderr, "INTERNAL ERROR: Failed to add archive header for '%s'!\n", line);
      fclose(buildf);
      return -1;
    }
  }
  
  fclose(buildf);
  
  FILE* outputf = fopen(outname, "w");
  
  if (outputf == NULL) {
    fprintf(stderr, "WRITE ERROR: Failed to open package build file '%s' for writing!\n", outname);
    return -1;
  }
  
  // The first, "blank" run only writes a placeholder of the headers
  if (pkg_archive_writeheaders(&archive, outputf, 1) != 0) {
    fprintf(stderr, "ERROR WRITING HEADERS: First run of pkg_archive_writeheaders failed!\n");
    return -1;
  }
  
  // The files themselves are written after the blank headers, but checksums &
  // sizes for the headers are calculated during writing.
  if (pkg_archive_writecontents(&archive, outputf, distd) != 0) {
    fprintf(stderr, "ERROR WRITING CONTENTS: The pkg_archive_writecontents function failed!\n");
    return -1;
  }
  
  // The output file is then closed and reopened, or on compatible systems fseek
  // or something like that can be used to return to the start of the file.
  
  fclose(outputf);
  
  outputf = fopen(outname, "w");
  
  if (outputf == NULL) {
    fprintf(stderr, "WRITE ERROR: Failed to open package build file '%s' for writing!\n", outname);
    return -1;
  }
  
  // Now at the start of the file again, we can write the final headers.
  
  if (pkg_archive_writeheaders(&archive, outputf, 0) != 0) {
    fprintf(stderr, "ERROR WRITING HEADERS: Second run of pkg_archive_writeheaders failed!\n");
    return -1;
  }
  
  // And it seems we have to write the contents properly this time.
  if (pkg_archive_writecontents(&archive, outputf, distd) != 0) {
    fprintf(stderr, "ERROR WRITING CONTENTS: The pkg_archive_writecontents function failed!\n");
    return -1;
  }
  
  // Then upon this final close the file should be finished (subject to
  // verification when installing later).
  fclose(outputf);
  
  if (srcd != cd) {
    free(srcd);
  }
  free(distd);
  free(outname);
  free(shortname);
  
  return 0;
}

char* pkg_simplefname(char* name) {
  char* laststart = name;
  while (*name) {
    if ((*name == '/' || *name == '\\') && *(name+1) != 0) {
      laststart = name+1;
    }
    name++;
  }
  laststart = strdup(laststart);
  if (laststart == NULL) {
    return NULL;
  }
  for (int i = 0; laststart[i] != 0; i++) {
    if (laststart[i] == '.' || laststart[i] == '/' || laststart[i] == '\\') {
      laststart[i] = 0;
      return laststart;
    }
  }
  return laststart;
}

int pkg_job_install(pkg_job_t* job, char* name) {
  int n = strlen(name);
  if (n < 5 || name[n-4] != '.' || name[n-3] != 'p' || name[n-2] != 'k' || name[n-1] != 'g') {
    fprintf(stderr, "BAD FILENAME: Installable package file must end in .pkg\n");
    return -1;
  }
  
  pkg_archive_t archive;
  if (pkg_archive_init(&archive) != 0) {
    fprintf(stderr, "INTERNAL ERROR: Failed to initialise archive structure!\n");
    return -1;
  }
  
  FILE* inputf = fopen(name, "rb");
  
  if (inputf == NULL) {
    fprintf(stderr, "READ ERROR: Failed to open package file '%s' for reading!\n", name);
    return -1;
  }
  
  if (pkg_archive_readheaders(&archive, inputf) != 0) {
    fprintf(stderr, "FILE ERROR: Failed to read package headers!\n");
    return -1;
  }
  
  if (pkg_archive_extractcontents(&archive, inputf, job->installroot) != 0) {
    fprintf(stderr, "FILE ERROR: Failed to extract package contents!\n");
    return -1;
  }
  
  fclose(inputf);
  
  if (job->usedb) {
  
    inputf = fopen(name, "rb");
    
    if (inputf == NULL) {
      fprintf(stderr, "READ ERROR: Failed to open package file '%s' for reading!\n", name);
      return -1;
    }
    
    char* simplename = pkg_simplefname(name);
    char* dbfile = pkg_ezstrcat(job->dbroot, simplename);
    dbfile = pkg_ezstrcat(dbfile, ".pkh");
    
    
    fprintf(stderr, "Creating installation record '%s'...\n", dbfile);
    
    FILE* outputf = fopen(dbfile, "w");
    if (outputf == NULL) {
      pkg_mkdir(job->dbroot);
      outputf = fopen(dbfile, "w");
      if (outputf == NULL) {
        fprintf(stderr, "READ ERROR: Failed to open db file '%s' for writing!\n", dbfile);
        return -1;
      }
    }
    
    size_t nread;
    do {
      nread = fread(pkg_linebuffer, 1, PKG_BUFFERSIZE, inputf);
      if (nread > 0) {
        size_t nwritten = fwrite(pkg_linebuffer, 1, nread, outputf);
        if (nwritten != nread) {
          fprintf(stderr, "WRITE ERROR: Failed while writing db file '%s'!\n", dbfile);
          fclose(inputf);
          fclose(outputf);
          return -1;
        }
      }
    } while (nread > 0);
    
    fclose(inputf);
    fclose(outputf);
    
    dbfile = pkg_ezstrcat(job->dbroot, "pkglist.txt");
    
    outputf = fopen(dbfile, "a");
    if (outputf == NULL) {
      fprintf(stderr, "Creating database index file '%s'...\n", dbfile);
      outputf = fopen(dbfile, "w");
      if (outputf == NULL) {
        fprintf(stderr, "READ ERROR: Failed to open db file '%s' for writing!\n", dbfile);
        return -1;
      }
    }
    
    fprintf(outputf, "%s\n", simplename);
    
    fclose(outputf);
  }
  
  return 0;
}

int pkg_job_remove(pkg_job_t* job, char* name) {
  int n = strlen(name);
  FILE* inputf = NULL;
  char* dbfile = NULL;
  if (n < 5 || name[n-4] != '.' || name[n-3] != 'p' || name[n-2] != 'k' || (name[n-1] != 'g' && name[n-1] != 'h')) {
    if (job->usedb) {
      
      char* simplename = pkg_simplefname(name);
      dbfile = pkg_ezstrcat(job->dbroot, simplename);
      dbfile = pkg_ezstrcat(dbfile, ".pkh");
    
      inputf = fopen(dbfile, "rb");
      
      if (inputf == NULL) {
        fprintf(stderr, "READ ERROR: Failed to open package file '%s' for reading!\n", dbfile);
        return -1;
      }
    } else {
      fprintf(stderr, "BAD FILENAME: Installable package file must end in .pkg or .pkh if db is explicitly disabled\n");
      return -1;
    }
  } else {
    inputf = fopen(name, "rb");
    
    if (inputf == NULL) {
      fprintf(stderr, "READ ERROR: Failed to open package file '%s' for reading!\n", name);
      return -1;
    }
  }
  
  pkg_archive_t archive;
  if (pkg_archive_init(&archive) != 0) {
    fprintf(stderr, "INTERNAL ERROR: Failed to initialise archive structure!\n");
    return -1;
  }
  
  if (pkg_archive_readheaders(&archive, inputf) != 0) {
    fprintf(stderr, "FILE ERROR: Failed to read package headers!\n");
    return -1;
  }
  
  if (pkg_archive_uninstallcontents(&archive, inputf, job->installroot) != 0) {
    fprintf(stderr, "FILE ERROR: Failed to uninstall package contents!\n");
    return -1;
  }
  
  fclose(inputf);
  
  // Clean up the .pkh file after uninstalling the packaged files
  if (dbfile) {
    fprintf(stderr, "Removing installation record '%s'...\n", dbfile);
    pkg_remove(dbfile, 0);
  }
  
  return 0;
}

int pkg_job_sum(pkg_job_t* job, char* name) {
  pkg_sum_t sum;
  //fprintf(stderr, "Checking the sum of '%s'...\n", name);
  
  if (pkg_sum_init(&sum, name, 0) != 0) {
    fprintf(stderr, "INTERNAL ERROR: pkg_sum_init failed for '%s'!\n", name);
    return -1;
  }
  FILE* inf = fopen(name, "rb");
  
  if (inf == NULL) {
    fprintf(stderr, "BAD FILE: Failed to open file '%s' for reading!\n", name);
    return -1;
  }
  
  size_t nread = 0;
  do {
    nread = fread(pkg_linebuffer, 1, PKG_BUFFERSIZE, inf);
    if (pkg_sum_append(&sum, pkg_linebuffer, nread) != 0) {
      fprintf(stderr, "INTERNAL ERROR: pkg_sum_append failed for '%s'!\n", name);
      fclose(inf);
      return -1;
    }
  } while (nread == PKG_BUFFERSIZE);
  
  fclose(inf);
  
  printf("%s checksum64=0x%llX checksum32=0x%X size=%llu [namesum=0x%X]\n", sum.name, sum.checksum64, sum.checksum32, sum.extrsize, pkg_cstringhash32(sum.name));
  
  return 0;
}

int pkg_compress_f2f(FILE* inf, FILE* outf, size_t size, pkg_sum_t* sum) {
  size_t nread = 0;
  do {
    nread = fread(pkg_linebuffer, 1, PKG_BUFFERSIZE, inf);
    if (sum != NULL && pkg_sum_append(sum, pkg_linebuffer, nread) != 0) {
      fprintf(stderr, "INTERNAL ERROR: pkg_sum_append failed during compression!\n");
      return -1;
    }
    
    if (nread > 0) {
      printf(".");
      fflush(stdout);
      pkg_sum_t blocksum;
      if (pkg_sum_init(&blocksum, "<TMP-BLOCK>", 0) != 0) {
        fprintf(stderr, "INTERNAL ERROR: pkg_sum_init failed!\n");
        return -1;
      }
      
      size_t ncompressed = pkg_compressv1(pkg_linebuffer, nread, pkg_comprbuffer, PKG_COMPRBUFSIZE);
      
      if (ncompressed < 1 || ncompressed > PKG_COMPRBUFSIZE || ncompressed >= 64*1024) {
        fprintf(stderr, "INTERNAL ERROR: pkg_compressv1 failed!\n");
        return -1;
      }
      
      if (pkg_sum_append(&blocksum, pkg_comprbuffer, ncompressed) != 0) {
        fprintf(stderr, "INTERNAL ERROR: pkg_sum_append failed during compression!\n");
        return -1;
      }
      
      unsigned char hdr[4];
      hdr[0] = 0xCF;
      hdr[1] = 0x11;
      hdr[2] = (unsigned char) ncompressed;
      hdr[3] = (unsigned char) (ncompressed >> 8);
      if (fwrite(hdr, 1, 4, outf) != 4) {
        fprintf(stderr, "WRITE ERROR: Failed while writing compressed output!\n");
        return -1;
      }
      
      if (fwrite(pkg_comprbuffer, 1, ncompressed, outf) != ncompressed) {
        fprintf(stderr, "WRITE ERROR: Failed while writing compressed output!\n");
        return -1;
      }
      
      if (pkg_archive_writef32(NULL, blocksum.checksum32, outf, NULL) != 4) {
        fprintf(stderr, "WRITE ERROR: Failed while writing compressed output!\n");
        return -1;
      }
    }
  } while (nread == PKG_BUFFERSIZE);
  return 0;
}

int pkg_decompress_f2f(FILE* inf, FILE* outf, size_t size, pkg_sum_t* sum) {
  size_t nread = 0;
  size_t totalread = 0;
  do {
    if (size != -1 && nread + 8 > size) {
      fprintf(stderr, "INTERNAL ERROR: Buffer has trailing junk not long enough to be a compressed chunk!\n");
      return -1;
    }
    unsigned char hdr[4];
    nread = fread(hdr, 1, 4, inf);
    if (nread == 0 && size == -1) {
      // Finished reading input stream until the end, return as normal.
      return 0;
    }
    if (nread != 4) {
      fprintf(stderr, "READ ERROR: Failed at block header while decompressing file stream!\n");
      return -1;
    }
    
    if (hdr[0] != 0xCF || hdr[1] != 0x11) {
      fprintf(stderr, "VERIFICATION ERROR: Magic number at start of compressed data block doesn't match for compressed data!\n");
      return -1;
    }
    size_t ncompressed = (((size_t) hdr[2]) & 0xFF) | ((((size_t) hdr[3]) & 0xFF) << 8);
    fprintf(stderr, "Reading %d bytes (compressed)...\n", ncompressed);
    if (size != -1 && nread + 4 + ncompressed > size) {
      fprintf(stderr, "INTERNAL ERROR: Buffer size does not fit compressed chunk size!\n");
      return -1;
    }
      
    if (sum != NULL && pkg_sum_append(sum, pkg_linebuffer, nread) != 0) {
      fprintf(stderr, "INTERNAL ERROR: pkg_sum_append failed during compression!\n");
      return -1;
    }
    
    if (nread > 0) {
      printf(".");
      fflush(stdout);
      pkg_sum_t blocksum;
      if (pkg_sum_init(&blocksum, "<TMP-BLOCK>", 0) != 0) {
        fprintf(stderr, "INTERNAL ERROR: pkg_sum_init failed!\n");
        return -1;
      }
      
      nread += fread(pkg_comprbuffer, 1, ncompressed, inf);
      
      if (nread != 4 + ncompressed) {
        fprintf(stderr, "READ ERROR: Failed mid-block while decompressing file stream!\n");
        return -1;
      }
      
      size_t ndecompressed = pkg_decompressv1(pkg_comprbuffer, ncompressed, pkg_linebuffer, PKG_BUFFERSIZE);
      
      if (ndecompressed < 1 || ndecompressed > PKG_BUFFERSIZE || ndecompressed >= 64*1024) {
        fprintf(stderr, "INTERNAL ERROR: pkg_decompressv1 failed!\n");
        return -1;
      }
      
      if (pkg_sum_append(&blocksum, pkg_linebuffer, ndecompressed) != 0) {
        fprintf(stderr, "INTERNAL ERROR: pkg_sum_append failed during compression!\n");
        return -1;
      }
      
      unsigned int comprsum;
      
      if (pkg_archive_readf32(NULL, &comprsum, inf, NULL) != 4) {
        fprintf(stderr, "READ ERROR: Failed while reading compressed input!\n");
        return -1;
      }
      
      size_t nwritten = fwrite(pkg_linebuffer, 1, ndecompressed, outf);
      if (nwritten != ndecompressed) {
        fprintf(stderr, "WRITE ERROR: Failed mid-block while decompressing file stream! Tried to write %d bytes but only wrote %d!\n", (int) ndecompressed, (int) nwritten);
        return -1;
      }
      
      nread += 4;
      totalread += nread;
    }
  } while (nread > 0 && (size == -1 || totalread < size));
  return 0;
}

int pkg_job_compress(pkg_job_t* job, char* name) {
  pkg_sum_t sum;
  char* outname = pkg_ezstrcat(name, ".cv1");
  
  if (pkg_sum_init(&sum, outname, 0) != 0) {
    fprintf(stderr, "INTERNAL ERROR: pkg_sum_init failed for '%s'!\n", name);
    free(outname);
    return -1;
  }
  
  FILE* inf = fopen(name, "rb");
  
  if (inf == NULL) {
    fprintf(stderr, "BAD FILE: Failed to open file '%s' for reading!\n", name);
    free(outname);
    return -1;
  }
  
  FILE* outf = fopen(outname, "w");
  
  if (outf == NULL) {
    fprintf(stderr, "WRITE ERROR: Failed to open file '%s' for writing!\n", outname);
    free(outname);
    fclose(inf);
    return -1;
  }
  
  int result = pkg_compress_f2f(inf, outf, -1, &sum);
  
  fclose(inf);
  fclose(outf);
  
  if (result < 0) {
    fprintf(stderr, "COMPRESSION FAILED: pkg_compress_f2f returned non-zero result!\n");
    return -1;
  }
  
  printf("%s checksum64=0x%llX checksum32=0x%X size=%llu [namesum=0x%X]\n", sum.name, sum.checksum64, sum.checksum32, sum.extrsize, pkg_cstringhash32(sum.name));
  
  return 0;
}

int pkg_job_decompress(pkg_job_t* job, char* name) {
  pkg_sum_t sum;
  int namelen = (int) strlen(name);
  
  if (namelen < 4 || name[namelen-4] != '.' || name[namelen-3] != 'c' || name[namelen-2] != 'v' || name[namelen-1] != '1') {
    fprintf(stderr, "BAD FILE: expected a file with extension .cv1 but got '%s'!\n", name);
  }
  char* outname = strdup(name);
  outname[namelen-4] = 0; // Remove .cv1 extension
    
  if (pkg_sum_init(&sum, outname, 0) != 0) {
    fprintf(stderr, "INTERNAL ERROR: pkg_sum_init failed for '%s'!\n", name);
    free(outname);
    return -1;
  }
  
  FILE* inf = fopen(name, "rb");
  
  if (inf == NULL) {
    fprintf(stderr, "BAD FILE: Failed to open file '%s' for reading!\n", name);
    free(outname);
    return -1;
  }
  
  FILE* outf = fopen(outname, "w");
  
  if (outf == NULL) {
    fprintf(stderr, "WRITE ERROR: Failed to open file '%s' for writing!\n", outname);
    free(outname);
    fclose(inf);
    return -1;
  }
  
  int result = pkg_decompress_f2f(inf, outf, -1, &sum);
  
  fclose(inf);
  fclose(outf);
  
  if (result < 0) {
    fprintf(stderr, "DECOMPRESSION FAILED: pkg_decompress_f2f returned non-zero result!\n");
    return -1;
  }
  
  printf("%s checksum64=0x%llX checksum32=0x%X size=%llu [namesum=0x%X]\n", sum.name, sum.checksum64, sum.checksum32, sum.extrsize, pkg_cstringhash32(sum.name));
  
  return 0;
}

int pkg_job_subrun(pkg_job_t* job, char* name) {
  switch (job->jobtype) {
    case PKG_JOBTYPE_BUILD:
      return pkg_job_build(job, name);
    case PKG_JOBTYPE_INSTALL:
      return pkg_job_install(job, name);
    case PKG_JOBTYPE_REMOVE:
      return pkg_job_remove(job, name);
    case PKG_JOBTYPE_SUM:
      return pkg_job_sum(job, name);
    case PKG_JOBTYPE_COMPRESS:
      return pkg_job_compress(job, name);
    case PKG_JOBTYPE_DECOMPRESS:
      return pkg_job_decompress(job, name);
    default:
      return -1;
  }
}

const char* pkg_job_typename(pkg_job_t* job) {
  switch (job->jobtype) {
    case PKG_JOBTYPE_BUILD:
      return "build";
    case PKG_JOBTYPE_INSTALL:
      return "install";
    case PKG_JOBTYPE_REMOVE:
      return "remove";
    case PKG_JOBTYPE_SUM:
      return "sum";
    case PKG_JOBTYPE_COMPRESS:
      return "compress";
    case PKG_JOBTYPE_DECOMPRESS:
      return "decompress";
    default:
      return -1;
  }
}

int pkg_job_run(pkg_job_t* job) {
  int i;
  printf("Running job type '%s' with %d entries\n", pkg_job_typename(job), job->nnames);
  for (i = 0; i < job->nnames; i++) {
    int subresult = pkg_job_subrun(job, job->names[i]);
    if (subresult != 0) {
      fprintf(stderr, "JOB FAILED!\n");
      return subresult;
    }
  }
  fprintf(stderr, "JOB DONE.\n");
  return 0;
}

void formatinfo(FILE* out) {
  fprintf(out, ".pkb (package build file) format:\n");
  fprintf(out, "  The build command runs on package build files (.pkb), creating associated .pkg package files.\n");
  fprintf(out, "  A `.pkb` file must exist in the root of a package source directory or a `pkgs` subdirectory.\n");
  fprintf(out, "  One `.pkb` file exists per package built from a source directory.\n");
  fprintf(out, "  This file lists packageable files, normally placed in a 'dist' subdirectory.\n");
  fprintf(out, "  A single `.pkb` file contains one such line for every installed file.\n");
  fprintf(out, "  Some lines are versioning or entries for directories, plain files can just be listed.\n\n");
  fprintf(out, ".pkg (package file) format:\n");
  fprintf(out, "  Package files use a custom format (not zip or tar) making management seamless.\n");
  fprintf(out, "  Full headers & checksums are placed at the start of the package file.\n");
  fprintf(out, "  Data for each installable file follows and can be either raw or compressed (unfinished/testing).\n\n");
  fprintf(out, ".cv1 (compression version 1) format:\n");
  fprintf(out, "  This is a simple stream-of-blocks style compression format with per-block integrity checking.\n");
  fprintf(out, "  These files are used to compress individual files, but the format itself may be used elsewhere.\n\n");
  fprintf(out, ".pkh (package header) format:\n");
  fprintf(out, "  This is the same as the `.pkg` format except only the headers are used to track installed files.\n");
  fprintf(out, "  These files are kept automatically by the package manager for it's internal database.\n\n");
  fprintf(out, "pkglist.txt format:\n");
  fprintf(out, "  This is just a list of possibly-valid packages which might be installed on the system.\n");
  fprintf(out, "  A package is considered installed if it's listed here AND has a valid .pkh file.\n");
  fprintf(out, "  These files are kept automatically by the package manager for it's internal database.\n\n");
}


int usage(int argc, char** argv, int argi, char* error) {
  FILE* out = error ? stderr : stdout;
  char* prgname = argv[0];
  fprintf(out, "USAGE:\n\n");
  fprintf(out, "  %s build <project1.pkb> [<project2.pkb>...]\n  ^ to build .pkg files from .pkb files in a source/build directory\n\n", prgname);
  fprintf(out, "  %s [options] install <package1.pkg> [<package2.pkg>...]\n  ^ to install one or more .pkg files\n\n", prgname);
  fprintf(out, "  %s [options] remove <pkgname> [<pkgname2>...]\n  ^ to un-install one or more packages\n\n", prgname);
  fprintf(out, "  %s sum <file1> [<file2>...]\n  ^ to print checksum and size info of regular files\n\n", prgname);
  fprintf(out, "  %s compress <file1> [<file2>...]\n  ^ writes compressed version of file with .cv1 extension added\n\n", prgname);
  fprintf(out, "  %s decompress <file1.cv1> [<file2.cv1>...]\n  ^ writes decompressed version of file with .cv1 extension removed\n\n", prgname);
  fprintf(out, "  %s help\n  ^ displays this usage information\n\n", prgname);
  
  fprintf(out, "OPTIONS:\n\n");
  fprintf(out, "  --root <installdir>\n  ^ installs/manages packages inside a directory (uses it as a root directory)\n\n");
  fprintf(out, "  --dbroot <configdir>\n  ^ keeps database of installed packages in this directory, defaults to <installdir>/.pkgs/\n\n");
  fprintf(out, "  --nodb\n ^ doesn't use a database of installed packages\n\n");
  
  formatinfo(out);
  
  fprintf(out, "DEMO INSTRUCTIONS:\n\n");
  fprintf(out, "  1. compile slabpkg.c\n  2. place the executable in a subdirectory named `dist`\n  3. create a file `slabpkg.pkb` with a line `/slabpkg` (executable path within `dist`)\n  4. run `./dist/slabpkg build slabpkg.pkb`\n  5. work out how to install it (try --root ... install)\n\n");
  
  if (error) {
    fprintf(out, "ERROR:\n  %s\n", error);
    return -1;
  } else {
    return 0;
  }
}

int main(int argc, char** argv) {
  pkg_job_t job;
  int argi = 1;
  printf("SecureLang Package Manager 0.0.2\nThis currently doesn't include proper versioning or dependencies.\n");
  
  if (argc < 2) {
    return usage(argc, argv, argi, "Expected command (build, install, ...)");
  }
  
  job.installroot = NULL;
  job.dbroot = NULL;
  job.usedb = 1;
  
  job.verbose = 0;
  job.target = "/";
  
  int foundoption;
  do {
    foundoption = 0;
    if (!strcmp(argv[argi], "--root")) {
      argi++;
      if (argi >= argc) {
        return usage(argc, argv, argi, "Expected a directory name following the --root option.");
      }
      job.installroot = strdup(argv[argi]);
      argi++;
    } else if (!strcmp(argv[argi], "--dbroot")) {
      argi++;
      if (argi >= argc) {
        return usage(argc, argv, argi, "Expected a directory name following the --dbroot option.");
      }
      job.installroot = strdup(argv[argi]);
      argi++;
    } else if (!strcmp(argv[argi], "--nodb")) {
      argi++;
      job.usedb = 0;
    }
  } while (foundoption);
  
  if (job.installroot == NULL) {
    job.installroot = strdup("/");
  }
  
  if (job.usedb && job.dbroot == NULL) {
    job.dbroot = pkg_ezstrcat(job.installroot, ".pkgs/");
  }
  
  while(job.installroot[strlen(job.installroot) - 1] == '/' || job.installroot[strlen(job.installroot) - 1] == '\\') {
    job.installroot[strlen(job.installroot) - 1] = 0;
  }
  
  if (!strcmp(argv[argi], "build")) {
    argi++;
    if (argi >= argc) {
      return usage(argc, argv, argi, "Expected one or more package build files (.pkb) for build command.\n");
    }
    job.names = argv + argi;
    job.nnames = argc - argi;
    job.jobtype = PKG_JOBTYPE_BUILD;
    return pkg_job_run(&job);
  } else if (!strcmp(argv[argi], "install")) {
    argi++;
    if (argi >= argc) {
      return usage(argc, argv, argi, "Expected one or more package files (.pkg) for install command.\n");
    }
    job.names = argv + argi;
    job.nnames = argc - argi;
    job.jobtype = PKG_JOBTYPE_INSTALL;
    return pkg_job_run(&job);
  } else if (!strcmp(argv[argi], "remove")) {
    argi++;
    if (argi >= argc) {
      return usage(argc, argv, argi, "Expected one or more package names for remove command.\n");
    }
    job.names = argv + argi;
    job.nnames = argc - argi;
    job.jobtype = PKG_JOBTYPE_REMOVE;
    return pkg_job_run(&job);
  } else if (!strcmp(argv[argi], "sum")) {
    argi++;
    if (argi >= argc) {
      return usage(argc, argv, argi, "Expected one or more files for sum command.\n");
    }
    job.names = argv + argi;
    job.nnames = argc - argi;
    job.jobtype = PKG_JOBTYPE_SUM;
    return pkg_job_run(&job);
  } else if (!strcmp(argv[argi], "compress")) {
    argi++;
    if (argi >= argc) {
      return usage(argc, argv, argi, "Expected one or more files for compress command.\n");
    }
    job.names = argv + argi;
    job.nnames = argc - argi;
    job.jobtype = PKG_JOBTYPE_COMPRESS;
    return pkg_job_run(&job);
  } else if (!strcmp(argv[argi], "decompress")) {
    argi++;
    if (argi >= argc) {
      return usage(argc, argv, argi, "Expected one or more compressed files (.cv1) for decompress command.\n");
    }
    job.names = argv + argi;
    job.nnames = argc - argi;
    job.jobtype = PKG_JOBTYPE_DECOMPRESS;
    return pkg_job_run(&job);
  } else if (!strcmp(argv[argi], "help")) {
    return usage(argc, argv, argi, NULL);
  } else {
    return usage(argc, argv, argi, "Unrecognised command");
  }
}