Compiler: Generate .scp files as part of the build process.

Stolen from SM (https://github.com/alliedmodders/sourcemod/pull/112).

compiler/libpc300/AMBuilder

@@ -1,9 +1,50 @@
 # vim: set sts=2 ts=8 sw=2 tw=99 et ft=python:
 import os.path
 
+# Build the packing binary garbage.
+scpack = AMXX.Program(builder, 'scpack')
+if builder.target_platform == 'windows':
+  scpack.compiler.linkflags.remove('/SUBSYSTEM:WINDOWS')
+  scpack.compiler.linkflags.append('/SUBSYSTEM:CONSOLE')
+scpack.sources = ['scpack.c']
+scpack = builder.Add(scpack)
+
+# Generate pack files.
+packed_files = ['sc5', 'sc7']
+packed_includes = []
+for packed_file in packed_files:
+  # The absolute path to sc5-in.scp etc.
+  in_path = os.path.join(builder.currentSourcePath, '{0}-in.scp'.format(packed_file))
+
+  # The output file relative to the output folder, i.e. sourcepawn/compiler/sc5.scp.
+  out_path = os.path.join(builder.buildFolder, '{0}.scp'.format(packed_file))
+
+  # The absolute path to the build folder, i.e. /Users/.../sourcepawn/compiler.
+  build_folder = os.path.join(builder.buildPath, builder.buildFolder)
+
+  # scpack runs in ./sourcepawn/compiler/scpack/ so we build relative paths
+  # from there.
+  scpack_argv = [
+    os.path.join(builder.buildPath, scpack.binary.path),
+    os.path.relpath(in_path, build_folder),
+    os.path.relpath(os.path.join(builder.buildPath, out_path), build_folder),
+  ]
+
+  _, (entry,) = builder.AddCommand(
+    inputs = [scpack.binary, in_path],
+    argv = scpack_argv,
+    outputs = ['{0}.scp'.format(packed_file)],
+  )
+  packed_includes += [entry]
+  
 binary = AMXX.Library(builder, 'amxxpc32')
 
-binary.compiler.includes += [builder.currentSourcePath]
+binary.compiler.includes += [
+  builder.currentSourcePath,
+  os.path.join(builder.buildPath, builder.buildFolder),
+]
+
+binary.compiler.sourcedeps += packed_includes
 
 if builder.target_platform in ['mac', 'linux']:
   binary.compiler.defines += ['ENABLE_BINRELOC']

compiler/libpc300/sc5.c

@@ -41,7 +41,7 @@
   #pragma warning(disable:4125)  /* decimal digit terminates octal escape sequence */
 #endif
 
-#include "sc5.scp"
+#include <sc5.scp>
 
 #if defined _MSC_VER
   #pragma warning(pop)

compiler/libpc300/sc7.c

@@ -51,7 +51,7 @@
   #pragma warning(disable:4125)  /* decimal digit terminates octal escape sequence */
 #endif
 
-#include "sc7.scp"
+#include <sc7.scp>
 
 #if defined _MSC_VER
   #pragma warning(pop)

compiler/libpc300/scpack.c

@@ -0,0 +1,450 @@
+/* compress.c -- Byte Pair Encoding compression */
+/* Copyright 1996 Philip Gage */
+
+/* This program appeared in the September 1997 issue of
+ * C/C++ Users Journal. The original source code may still
+ * be found at the web site of the magazine (www.cuj.com).
+ *
+ * It has been modified by me (Thiadmer Riemersma) to
+ * compress only a section of the input file and to store
+ * the compressed output along with the input as "C" strings.
+ *
+ * Compiling instructions:
+ *  Borland C++ 16-bit (large memory model is required):
+ *      bcc -ml scpack.c
+ *
+ *  Watcom C/C++ 32-bit:
+ *      wcl386 scpack.c
+ *
+ *  GNU C (Linux), 32-bit:
+ *      gcc scpack.c -o scpack
+ */
+
+#include <assert.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if UINT_MAX > 0xFFFFU
+  #define MAXSIZE 1024*1024L
+#else
+  #define MAXSIZE UINT_MAX      /* Input file buffer size */
+#endif
+#define HASHSIZE 8192   /* Hash table size, power of 2 */
+#define THRESHOLD   3   /* Increase for speed, min 3 */
+
+#define START_TOKEN "#ifdef SCPACK" /* start reading the buffer here */
+#define NAME_TOKEN  "#define SCPACK_TABLE"
+#define SEP_TOKEN   "#define SCPACK_SEPARATOR"
+#define TERM_TOKEN  "#define SCPACK_TERMINATOR"
+#define TEMPFILE    "~SCPACK.TMP"
+static char tablename[32+1] = "scpack_table";
+static char separator[16]=",";
+static char terminator[16]="";
+
+int compress(unsigned char *buffer, unsigned buffersize, unsigned char pairtable[128][2])
+{
+  unsigned char *left, *right, *count;
+  unsigned char a, b, bestcount;
+  unsigned i, j, index, bestindex, code=128;
+
+  /* Dynamically allocate buffers and check for errors */
+  left = (unsigned char *)malloc(HASHSIZE);
+  right = (unsigned char *)malloc(HASHSIZE);
+  count = (unsigned char *)malloc(HASHSIZE);
+  if (left==NULL || right==NULL || count==NULL) {
+    printf("Error allocating memory\n");
+    exit(1);
+  }
+
+  /* Check for errors */
+  for (i=0; i<buffersize; i++)
+    if (buffer[i] > 127) {
+      printf("This program works only on text files (7-bit ASCII)\n");
+      exit(1);
+    }
+
+  memset(pairtable, 0, 128*2*sizeof(char));
+
+  do {  /* Replace frequent pairs with bytes 128..255 */
+
+    /* Enter counts of all byte pairs into hash table */
+    memset(count,0,HASHSIZE);
+    for (i=0; i<buffersize-1; i++) {
+      a = buffer[i];
+      b = buffer[i+1];
+      /* ignore any pair with a '\0' */
+      if (a == 0 || b == 0)
+        continue;
+      index = (a ^ (b << 6)) & (HASHSIZE-1);
+      while ((left[index] != a || right[index] != b) &&
+             count[index] != 0)
+        index = (index + 1) & (HASHSIZE-1);
+      left[index] = a;
+      right[index] = b;
+      if (count[index] < 255)
+        count[index] += (unsigned char)1;
+    }
+
+    /* Search hash table for most frequent pair */
+    bestcount = THRESHOLD - 1;
+    for (i=0; i<HASHSIZE; i++) {
+      if (count[i] > bestcount) {
+        bestcount = count[i];
+        bestindex = i;
+      }
+    }
+
+    /* Compress if enough occurrences of pair */
+    if (bestcount >= THRESHOLD) {
+
+      /* Add pair to table using code as index */
+      a = pairtable[code-128][0] = left[bestindex];
+      b = pairtable[code-128][1] = right[bestindex];
+
+      /* Replace all pair occurrences with unused byte */
+      for (i=0, j=0; i<buffersize; i++, j++)
+        if (a == buffer[i] && b == buffer[i+1]) {
+          buffer[j] = (unsigned char)code;
+          ++i;
+        }
+        else
+          buffer[j] = buffer[i];
+      buffersize = j;
+    }
+    else
+      break;
+  } while (++code < 255);
+
+  /* done */
+  free(left); free(right); free(count);
+  return buffersize;  /* return adjusted buffersize */
+}
+
+static int strmatch(char *str, char *token, int *indent)
+{
+  int i = 0;
+
+  /* skip whitespace */
+  while (*str==' ' || *str=='\t') {
+    str++;
+    i++;
+  } /* while */
+  if (strncmp(str,token,strlen(token))!=0)
+    return 0;
+  if (indent != NULL)
+    *indent = i;
+  return 1;
+}
+
+static void check_if(char *str,int linenr)
+{
+  if (strmatch(str,"#if",NULL)) {
+    printf("Error: \"#if...\" preprocessor statement should not be in SCPACK section "
+           "(line %d)\n", linenr);
+    exit(1);
+  } /* if */
+}
+
+static int check_tablename(char *str)
+{
+  int i;
+
+  if (strmatch(str,NAME_TOKEN,NULL)) {
+    str += strlen(NAME_TOKEN);
+    while (*str==' ' || *str=='\t')
+      str++;
+    for (i=0; i<(sizeof tablename - 1) && *str!='\0' && strchr(" \t\n",*str)==NULL; i++, str++)
+      tablename[i] = *str;
+    tablename[i] = '\0';
+    return 1;
+  } /* if */
+  return 0;
+}
+
+static int check_separator(char *str)
+{
+  int i;
+
+  if (strmatch(str,SEP_TOKEN,NULL)) {
+    str += strlen(SEP_TOKEN);
+    while (*str==' ' || *str=='\t')
+      str++;
+    for (i=0; i<(sizeof separator - 1) && *str!='\0' && strchr(" \t\n",*str)==NULL; i++, str++)
+      separator[i] = *str;
+    separator[i] = '\0';
+    return 1;
+  } /* if */
+
+  if (strmatch(str,TERM_TOKEN,NULL)) {
+    str += strlen(TERM_TOKEN);
+    while (*str==' ' || *str=='\t')
+      str++;
+    for (i=0; i<(sizeof terminator - 1) && *str!='\0' && strchr(" \t\n",*str)==NULL; i++, str++)
+      terminator[i] = *str;
+    terminator[i] = '\0';
+    return 1;
+  } /* if */
+
+  return 0;
+}
+
+/* readbuffer
+ * Reads in the input file and stores all strings in the
+ * section between "#ifdef SCPACK" and "#else" in a buffer.
+ * Only text that is between double quotes is added to the
+ * buffer; the \" escape code is handled. Multiple strings
+ * on one line are handled.
+ */
+unsigned readbuffer(FILE *input, unsigned char *buffer)
+{
+  char str[256];
+  unsigned buffersize;
+  int i,linenr;
+
+  linenr=0;
+  buffersize=0;
+
+  rewind(input);
+  while (!feof(input)) {
+    while (fgets(str,sizeof str,input)!=NULL) {
+      linenr++;
+      check_tablename(str);
+      check_separator(str);
+      if (strmatch(str,START_TOKEN,NULL))
+        break;
+    } /* while */
+    if (!strmatch(str,START_TOKEN,NULL))
+      return buffersize;  /* no (more) section found, quit */
+
+    while (fgets(str,sizeof str,input)!=NULL) {
+      linenr++;
+      check_if(str,linenr);
+      if (check_tablename(str))
+        printf("Error: table name definition should not be in SCPACK section (line %d)\n", linenr);
+      check_separator(str);
+      if (strmatch(str,"#else",NULL))
+        break;          /* done */
+      /* add to the buffer only what is between double quotes */
+      i=0;
+      do {
+        while (str[i]!='\0' && str[i]!='"')
+          i++;
+        if (str[i]=='"') {
+          /* we are in a string */
+          i++;
+          while (str[i]!='\0' && str[i]!='"') {
+            /* handle escape sequences */
+            if (str[i]=='\\') {
+              i++;
+              switch (str[i]) {
+              case 'a': /* alarm */
+                buffer[buffersize++]='\a';
+                i++;
+                break;
+              case 'b': /* backspace */
+                buffer[buffersize++]='\b';
+                i++;
+                break;
+              case 'f': /* form feed */
+                buffer[buffersize++]='\f';
+                i++;
+                break;
+              case 'n': /* newline */
+                buffer[buffersize++]='\n';
+                i++;
+                break;
+              case 'r': /* carriage return */
+                buffer[buffersize++]='\n';
+                i++;
+                break;
+              case 't': /* tab */
+                buffer[buffersize++]='\t';
+                i++;
+                break;
+              case '\'':
+                buffer[buffersize++]='\'';
+                i++;
+                break;
+              case '"':
+                buffer[buffersize++]='"';
+                i++;
+                break;
+              default:
+                // ??? octal character code escapes and hexadecimal escapes
+                //     not supported
+                printf("Unknown escape sequence '\\%c' on line %d\n",
+                       str[i], linenr);
+              } /* switch */
+            } else {
+              buffer[buffersize++]=str[i++];
+            } /* if */
+          } /* while */
+          if (str[i]=='"') {
+            buffer[buffersize++]='\0'; /* terminate each string */
+            i++;
+          } else {
+            printf("Error: unterminated string on line %d\n",linenr);
+          } /* if */
+        } /* if */
+      } while (str[i]!='\0');
+    } /* while - in SCPACK section */
+    /* put in another '\0' to terminate the section */
+    buffer[buffersize++]='\0';
+  } /* while - !feof(input) */
+  return buffersize;
+}
+
+static void write_pairtable(FILE *output, unsigned char pairtable[128][2], char *tablename)
+{
+  int i;
+
+  /* dump the pair table */
+  fprintf(output, "/*-*SCPACK start of pair table, do not change or remove this line */\n");
+  fprintf(output, "unsigned char %s[][2] = {", tablename);
+  for (i=0; i<128 && pairtable[i][0]!=0 && pairtable[i][1]!=0; i++) {
+    if ((i % 16)==0)
+      fprintf(output, "\n  ");
+    else
+      fprintf(output, " ");
+    fprintf(output, "{%d,%d}", pairtable[i][0], pairtable[i][1]);
+    /* check if something follows this pair */
+    if (i+1<128 && pairtable[i+1][0]!=0 && pairtable[i+1][1]!=0)
+      fprintf(output, ",");
+  } /* for */
+  fprintf(output, "\n};\n");
+  fprintf(output, "/*-*SCPACK end of pair table, do not change or remove this line */\n");
+}
+
+void writefile(FILE *input, FILE *output, unsigned char *buffer, unsigned buffersize, unsigned char pairtable[128][2])
+{
+  char str[256];
+  int insection, indent, needseparator;
+  unsigned char *bufptr;
+
+  bufptr = buffer;
+  insection = 0;
+
+  rewind(input);
+  while (!feof(input)) {
+    while (fgets(str,sizeof str,input)!=NULL) {
+      fprintf(output,"%s",str);
+      if (check_tablename(str)) {
+        write_pairtable(output, pairtable, tablename);
+        /* strip an existing pair table from the file */
+        if (fgets(str,sizeof str,input)!=NULL) {
+          if (strmatch(str,"/*-*SCPACK",NULL)) {
+            while (fgets(str,sizeof str,input)!=NULL)
+              if (strmatch(str,"/*-*SCPACK",NULL))
+                break;
+          } else {
+            fprintf(output,"%s",str);
+          } /* if */
+        } /* if */
+      } /* if */
+      if (strmatch(str,START_TOKEN,NULL))
+        insection = 1;
+      if (insection && strmatch(str,"#else",NULL))
+        break;
+    } /* while */
+    if (!strmatch(str,"#else",&indent))
+      return;           /* no (more) section found, quit */
+    insection=0;
+
+    /* dump the buffer as strings, separated with commas */
+    needseparator = 0;
+    while (*bufptr != '\0') {
+      assert((unsigned)(bufptr-buffer) < buffersize);
+      if (needseparator)
+        fprintf(output, "%s\n",separator);
+      fprintf(output, "%*c\"",indent+2,' ');
+      /* loop over string */
+      while (*bufptr != '\0') {
+        if (*bufptr<' ' || *bufptr >= 128 || *bufptr == '"' || *bufptr == '\\')
+          fprintf(output, "\\%03o", *bufptr);
+        else
+          fprintf(output, "%c", *bufptr);
+        bufptr++;
+      } /* while */
+      fprintf(output, "\"");
+      needseparator = 1;
+      bufptr++;           /* skip '\0' */
+    } /* while */
+    fprintf(output, "%s\n",terminator);
+    bufptr++;
+
+    /* skip the input file until the #endif section */
+    while (fgets(str,sizeof str,input)!=NULL) {
+      if (strmatch(str,"#endif",NULL)) {
+        fprintf(output,"%s",str);
+        break;          /* done */
+      } /* if */
+    } /* while */
+  } /* while - !feof(input) */
+}
+
+static void usage(void)
+{
+  printf("Usage: scpack <filename> [output file]\n");
+  exit(1);
+}
+
+int main(int argc, char **argv)
+{
+  FILE *in, *out;
+  unsigned char *buffer;
+  unsigned buffersize, orgbuffersize;
+  unsigned char pairtable[128][2];
+
+  if (argc < 2 || argc > 3)
+    usage();
+  if ((in=fopen(argv[1],"rt"))==NULL) {
+    printf("SCPACK: error opening input %s\n",argv[1]);
+    usage();
+  } /* if */
+  if (argc == 2) {
+    if ((out=fopen(TEMPFILE,"wt"))==NULL) {
+      printf("SCPACK: error opening temporary file %s\n",TEMPFILE);
+      usage();
+    } /* if */
+  } else {
+    if ((out=fopen(argv[2],"wt"))==NULL) {
+      printf("SCPACK: error opening output file %s\n",argv[2]);
+      usage();
+    } /* if */
+  } /* if */
+
+  buffer = (unsigned char *)malloc(MAXSIZE);
+  if (buffer == NULL) {
+    printf("SCPACK: error allocating memory\n");
+    return 1;
+  } /* if */
+  /* 1. read the buffer
+   * 2. compress the buffer
+   * 3. copy the file, insert the compressed buffer
+   */
+  buffersize = readbuffer(in, buffer);
+  orgbuffersize = buffersize;
+  if (buffersize > 0) {
+    buffersize = compress(buffer, buffersize, pairtable);
+    writefile(in, out, buffer, buffersize, pairtable);
+    printf("SCPACK: compression ratio: %ld%% (%d -> %d)\n",
+           100L-(100L*buffersize)/orgbuffersize, orgbuffersize, buffersize);
+  } else {
+    printf("SCPACK: no SCPACK section found, nothing to do\n");
+  } /* if */
+  fclose(out);
+  fclose(in);
+  /* let the new file replace the old file */
+  if (buffersize == 0) {
+    if (argc == 2)
+      remove(TEMPFILE);
+    else
+      remove(argv[2]);
+  } else if (argc == 2) {
+    remove(argv[1]);
+    rename(TEMPFILE,argv[1]);
+  } /* if */
+  return 0;
+}