Implement new dependency storage system

7 files changed, 349 insertions, 110 deletions

diff --git a/build.sh b/build.sh
index 0788e45..f1e1cf0 100755
--- a/build.sh
+++ b/build.sh
@@ -10,8 +10,10 @@ $CC $CFLAGS -o out/util.o -c src/util.c
 $CC $CFLAGS -o out/build.o -c src/build.c
 $CC $CFLAGS -o out/filepath.o -c src/filepath.c
 $CC $CFLAGS -o out/sha1.o -c src/sha1.c
+$CC $CFLAGS -o out/DSV.o -c src/DSV.c
 $CC $CFLAGS -o out/redo.o -c src/redo.c
-$CC -o out/redo out/redo.o out/util.o out/build.o out/filepath.o out/sha1.o $LDFLAGS
+$CC -o out/redo out/redo.o out/util.o out/build.o out/filepath.o out/sha1.o \
+       out/DSV.o $LDFLAGS
 
 # TODO: just for convenience, should be removed as soon as redo can build itself
 sudo install out/redo /usr/local/bin
diff --git a/out/redo.do b/out/redo.do
index 58e7152..ee098a0 100644
--- a/out/redo.do
+++ b/out/redo.do
@@ -1,5 +1,5 @@
 . ./config.sh
 
-DEPS="redo.o build.o util.o filepath.o sha1.o"
+DEPS="redo.o build.o util.o filepath.o sha1.o DSV.o"
 redo-ifchange $DEPS config.sh
 $CC -o $3 $DEPS $LDFLAGS
diff --git a/src/DSV.c b/src/DSV.c
new file mode 100644
index 0000000..91cc7ee
--- /dev/null
+++ b/src/DSV.c
@@ -0,0 +1,188 @@
+/* DSV.c
+ *
+ * Copyright (c) 2016 Tharre
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE file for details.
+ */
+
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "util.h"
+#include "dbg.h"
+#include "DSV.h"
+
+/* dest must be 2x src */
+size_t encode_string(char *dest, const char *src) {
+	char *start = dest, c;
+
+	while ((c = *(src++))) {
+		switch (c) {
+		case '\n':
+			*(dest++) = '\\';
+			*(dest++) = 'n';
+			break;
+		case '\\':
+			*(dest++) = '\\';
+			*(dest++) = '\\';
+			break;
+		case ':':
+			*(dest++) = '\\';
+			/* fall-through */
+		default:
+			*(dest++) = c;
+		}
+	}
+
+	*dest = '\0';
+	return dest - start;
+}
+
+static void decode_string(char *dest, const char *src, size_t len) {
+	for (size_t i = 0; i < len-1; ++i) {
+		char c = src[i];
+		if (c != '\\') {
+			*(dest++) = c;
+			continue;
+		}
+
+		/* check if we are at the end of the string */
+		if (*src == '\0') {
+			debug("DSV: ignored extra '\\' at end of string\n");
+			break;
+		}
+
+		switch (c = *(src++)) {
+		case 'n':
+			*(dest++) = '\n';
+			break;
+		default:
+			*(dest++) = c;
+		}
+	}
+
+	*dest = '\0';
+}
+
+void dsv_init(struct dsv_ctx *ctx, size_t fields_count) {
+	ctx->fields_count = fields_count;
+	ctx->fields = xmalloc(fields_count * sizeof(char*));
+	ctx->offset = 0;
+	ctx->bufsize = 1024;
+	ctx->buf = xmalloc(ctx->bufsize);
+	ctx->buflen = 0;
+}
+
+void dsv_free(struct dsv_ctx *ctx) {
+	free(ctx->fields);
+	free(ctx->buf);
+}
+
+int dsv_parse_next_line(struct dsv_ctx *context, const char *src, size_t len) {
+	char *newline = memchr(src, '\n', len);
+
+	if (!newline) {
+		context->status = E_NO_NEWLINE_FOUND;
+		return 1;
+	}
+
+	context->processed = newline-src+1;
+	if (context->processed == 1) {
+		debug("DSV: empty newline detected\n");
+		context->status = E_EMPTY_NEWLINE;
+		return 1;
+	}
+
+	char *start = (char*) src;
+	size_t i = 0;
+	ptrdiff_t size;
+
+	while (1) {
+		char *colon = memchr(start, ':', newline-start+1);
+		if (colon)
+			size = colon-start;
+		else
+			size = newline-start;
+
+		if (size <= 0) {
+			if (!size) {
+				debug("DSV: empty field detected\n");
+				context->status = E_EMPTY_FIELD;
+				goto error;
+			}
+
+			break;
+		}
+
+		char *buf = xmalloc(size+1);
+		decode_string(buf, start, size+1);
+		context->fields[i] = buf;
+
+		start += size + 1;
+		++i;
+		if (i > context->fields_count) {
+			debug("DSV: too many fields\n");
+			context->status = E_TOO_MANY_FIELDS;
+			goto error;
+		}
+	}
+
+	if (i+1 < context->fields_count) {
+		debug("DSV: too few fields (%zu)\n", i+1);
+		context->status = E_TOO_FEW_FIELDS;
+		goto error;
+	}
+
+	context->status = E_SUCCESS;
+	return 0;
+
+error:
+	for (size_t j = 0; j < i; ++j)
+		free(context->fields[j]);
+
+	return 1;
+}
+
+int dsv_parse_file(struct dsv_ctx *ctx, FILE *fp) {
+	size_t read = ctx->buflen;
+	if (!read) {
+		read = fread(ctx->buf, 1, ctx->bufsize, fp);
+		ctx->buflen = read;
+	}
+
+	while (read > 0) {
+		if (dsv_parse_next_line(ctx, ctx->buf+ctx->offset, read-ctx->offset)) {
+			if (ctx->status != E_NO_NEWLINE_FOUND)
+				return 1;
+
+			/* are we using the full buffer already? */
+			if (!ctx->offset) {
+				/* then make it bigger! */
+				ctx->bufsize *= 2;
+				ctx->buf = xrealloc(ctx->buf, ctx->bufsize);
+			} else {
+				/* then move stuff around */
+				memmove(ctx->buf, ctx->buf+ctx->offset, read-ctx->offset);
+				ctx->buflen -= ctx->offset;
+				ctx->offset = 0;
+			}
+
+			read = fread(ctx->buf+ctx->buflen, 1, ctx->bufsize-ctx->buflen, fp);
+			ctx->buflen += read;
+		} else {
+			ctx->offset += ctx->processed;
+			ctx->status = E_SUCCESS;
+			return 0;
+		}
+	}
+
+	if (ferror(fp)) {
+		ctx->status = E_FREAD_ERROR;
+		return 1;
+	}
+
+	return 1;
+}
diff --git a/src/DSV.h b/src/DSV.h
new file mode 100644
index 0000000..6b0c315
--- /dev/null
+++ b/src/DSV.h
@@ -0,0 +1,41 @@
+/* build.h
+ *
+ * Copyright (c) 2016 Tharre
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE file for details.
+ */
+
+#ifndef __RDSV_H__
+#define __RDSV_H__
+
+enum dsv_status {
+	E_SUCCESS = 0,
+	E_NO_NEWLINE_FOUND,
+	E_EMPTY_NEWLINE,
+	E_EMPTY_FIELD,
+	E_TOO_MANY_FIELDS,
+	E_TOO_FEW_FIELDS,
+	E_FREAD_ERROR,
+};
+
+struct dsv_ctx {
+	size_t processed;
+	size_t fields_count;
+	size_t offset;
+	size_t bufsize;
+	size_t buflen;
+
+	char **fields;
+	char *buf;
+
+	enum dsv_status status;
+};
+
+size_t encode_string(char *dest, const char *src);
+void dsv_init(struct dsv_ctx *context, size_t fields_count);
+void dsv_free(struct dsv_ctx *context);
+int dsv_parse_next_line(struct dsv_ctx *context, const char *src, size_t len);
+int dsv_parse_file(struct dsv_ctx *ctx, FILE *fp);
+
+#endif
diff --git a/src/build.c b/src/build.c
index 5f1de78..8903563 100644
--- a/src/build.c
+++ b/src/build.c
@@ -24,6 +24,7 @@
 #include "build.h"
 #include "util.h"
 #include "filepath.h"
+#include "DSV.h"
 #define _FILENAME "build.c"
 #include "dbg.h"
 
@@ -50,7 +51,7 @@ static char **parsecmd(char *cmd, size_t *i, size_t keep_free);
 static char *get_relpath(const char *target);
 static char *xrealpath(const char *path);
 static char *get_dep_path(const char *target);
-static void write_dep_header(dep_info *dep);
+static void write_dep_information(dep_info *dep);
 static int handle_ident(dep_info *dep, int ident);
 static int handle_c(dep_info *dep);
 static unsigned char *hash_file(const char *target);
@@ -70,7 +71,7 @@ static int build_target(dep_info *dep) {
 			if (!dep->hash)
 				dep->hash = hash_file(dep->target);
 
-			write_dep_header(dep);
+			write_dep_information(dep);
 			goto exit;
 		}
 
@@ -86,6 +87,12 @@ static int build_target(dep_info *dep) {
 	if (remove(dep->path) && errno != ENOENT)
 		fatal("redo: failed to remove %s", dep->path);
 
+	char *prereq = concat(2, dep->path, ".prereq");
+	if (remove(prereq) && errno != ENOENT)
+		fatal("redo: failed to remove %s", prereq);
+
+	free(prereq);
+
 	char *temp_output = concat(2, dep->target, ".redoing.tmp");
 
 	pid_t pid = fork();
@@ -154,7 +161,7 @@ static int build_target(dep_info *dep) {
 
 		free(old_hash);
 
-		write_dep_header(dep);
+		write_dep_information(dep);
 	} else {
 		if (remove(temp_output) && errno != ENOENT)
 			fatal("redo: failed to remove %s", temp_output);
@@ -168,16 +175,16 @@ static int build_target(dep_info *dep) {
 
 	if (!fexists(dep2.path)) {
 		dep2.hash = hash_file(doscripts->chosen);
-		write_dep_header(&dep2);
+		write_dep_information(&dep2);
 		free(dep2.hash);
 	}
 	free(dep2.path);
 
-	add_dep(doscripts->chosen, dep->target, 'c');
+	add_prereq(doscripts->chosen, dep->target, 'c');
 
 	/* redo-ifcreate on specific if general was chosen */
 	if (doscripts->general == doscripts->chosen)
-		add_dep(doscripts->specific, dep->target, 'e');
+		add_prereq(doscripts->specific, dep->target, 'e');
 
 	free(temp_output);
 exit:
@@ -332,38 +339,24 @@ static char *get_dep_path(const char *target) {
 }
 
 /* Declare that `parent` depends on `target`. */
-void add_dep(const char *target, const char *parent, int ident) {
-	char *dep_path = get_dep_path(parent);
-
-	if (strchr(target, '\n'))
-		die("redo: newlines in targets are not supported\n");
-
-	int fd = open(dep_path, O_WRONLY | O_APPEND);
-	if (fd < 0) {
-		if (errno != ENOENT)
-			fatal("redo: failed to open %s", dep_path);
-
-		/* no dependency record was found, so we create one */
-		fd = open(dep_path, O_WRONLY | O_APPEND | O_CREAT, 0644);
-		if (fd < 0)
-			fatal("redo: failed to open %s", dep_path);
-	}
+void add_prereq(const char *target, const char *parent, int ident) {
+	char *base_path = get_dep_path(parent);
+	char *dep_path = concat(2, base_path, ".prereq");
 
-	char garbage[HEADERSIZE];
-	memset(garbage, 'Z', HEADERSIZE);
-
-	/* skip header */
-	if (lseek(fd, 0, SEEK_END) < (off_t) HEADERSIZE)
-		pwrite(fd, garbage, HEADERSIZE, 0);
+	int fd = open(dep_path, O_WRONLY | O_APPEND | O_CREAT, 0644);
+	if (fd < 0)
+		fatal("redo: failed to open %s", dep_path);
 
 	char *reltarget = get_relpath(target);
-	int bufsize = strlen(reltarget) + 3;
+	size_t bufsize = strlen(reltarget)*2 + 3;
 	char *buf = xmalloc(bufsize);
+
 	buf[0] = ident;
-	buf[1] = '\t';
-	strcpy(buf+2, reltarget);
-	buf[bufsize-1] = '\n';
-	if (write(fd, buf, bufsize) < bufsize)
+	buf[1] = ':';
+	size_t encoded_len = encode_string(buf+2, reltarget);
+	buf[encoded_len+2] = '\n';
+
+	if (write(fd, buf, encoded_len+3) < (ssize_t) encoded_len+3)
 		fatal("redo: failed to write to %s", dep_path);
 
 	if (close(fd))
@@ -372,6 +365,7 @@ void add_dep(const char *target, const char *parent, int ident) {
 	free(buf);
 	free(reltarget);
 	free(dep_path);
+	free(base_path);
 }
 
 /* Hash the target file, returning a pointer to the heap allocated hash. */
@@ -416,26 +410,22 @@ static void hex_to_sha1(const char *s, unsigned char *sha1) {
 }
 
 /* Write the dependency information into the specified path. */
-static void write_dep_header(dep_info *dep) {
-	mode_t mode = S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH;
-	int out = open(dep->path, O_WRONLY | O_CREAT, mode);
-	if (out < 0)
+static void write_dep_information(dep_info *dep) {
+	FILE *fd = fopen(dep->path, "w+");
+	if (!fd)
 		fatal("redo: failed to open %s", dep->path);
 
-	char buf[60];
-	sprintf(buf, "%010u", atoi(getenv("REDO_MAGIC")));
-	buf[10] = '\t';
-	sha1_to_hex(dep->hash, buf+11);
-	buf[51] = '\t';
-	memset(buf+52, '-', 7);
-	if (dep->flags & DEP_SOURCE)
-		buf[52] = 'S';
-	buf[59] = '\n';
+	char hash[41];
+	sha1_to_hex(dep->hash, hash);
+	hash[40] = '\0';
+	char *flags = dep->flags & DEP_SOURCE ? "s" : "l";
 
-	if (write(out, buf, sizeof buf) < (ssize_t) sizeof buf)
-		fatal("redo: failed to write dependency record to '%s'", dep->path);
+	int magic = atoi(getenv("REDO_MAGIC"));
 
-	if (close(out))
+	if (fprintf(fd, "%s:%010u:%s\n", hash, magic, flags) < 0)
+		fatal("redo: failed to write to %s", dep->path);
+
+	if (fclose(fd))
 		fatal("redo: failed to close %s", dep->path);
 }
 
@@ -471,85 +461,103 @@ static int handle_ident(dep_info *dep, int ident) {
 static int handle_c(dep_info *dep) {
 	FILE *fp = fopen(dep->path, "rb");
 	if (!fp) {
-		if (errno == ENOENT)
+		if (errno == ENOENT) {
 			/* dependency record does not exist */
 			return build_target(dep);
-		else
+		} else {
 			fatal("redo: failed to open %s", dep->path);
+		}
 	}
 
-	char buf[FILENAME_MAX];
+	int retval = 0;
+	struct dsv_ctx ctx;
+	dsv_init(&ctx, 3);
 
-	if (fread(buf, 1, HEADERSIZE, fp) < HEADERSIZE)
-		fatal("redo: failed to read %zu bytes from %s", HEADERSIZE, dep->path);
+	if (dsv_parse_file(&ctx, fp)) {
+		retval = build_target(dep);
+		goto exit;
+	}
 
 	errno = 0;
-	buf[10] = '\0';
-	long magic = strtol(buf, NULL, 10);
-	if (errno)
-		return build_target(dep);
+	long magic = strtol(ctx.fields[1], NULL, 10);
+	if (errno) {
+		retval = build_target(dep);
+		goto exit;
+	}
 
 	if (!fexists(dep->target)) {
-		if (buf[52] == 'S') /* source flag set */
+		if (ctx.fields[2][0] == 's') {
 			/* target is a source and must not be rebuild */
-			return 1;
-		else
-			return build_target(dep);
+			retval = 1;
+			goto exit;
+		} else {
+			retval = build_target(dep);
+			goto exit;
+		}
 	}
 
-	if (magic == atoi(getenv("REDO_MAGIC")))
+	if (magic == atoi(getenv("REDO_MAGIC"))) {
 		/* magic number matches */
-		return 1;
+		retval = 1;
+		goto exit;
+	}
 
 	unsigned char old_hash[20];
-	buf[51] = '\0';
-	hex_to_sha1(buf+11, old_hash); /* TODO: error checking */
+	hex_to_sha1(ctx.fields[0], old_hash); /* TODO: error checking */
 	dep->hash = hash_file(dep->target);
 
-	if (memcmp(old_hash, dep->hash, 20))
-		return build_target(dep);
+	if (memcmp(old_hash, dep->hash, 20)) {
+		retval = build_target(dep);
+		goto exit;
+	}
 
-	char *ptr;
-	char *root = getenv("REDO_ROOT");
-	bool rebuild = false;
-
-	while (!feof(fp)) {
-		ptr = buf;
-
-		size_t read = fread(buf, 1, sizeof buf, fp);
-		if (ferror(fp))
-			fatal("redo: failed to read %zu bytes from descriptor", sizeof buf);
-
-		for (size_t i = 0; i < read; ++i) {
-			if (buf[i] != '\n')
-				continue;
-			buf[i] = '\0';
-			if (!is_absolute(&ptr[2])) {
-				/* if our path is relative we need to prefix it with the
-				   root project directory or the path will be invalid */
-				char *abs = concat(3, root, "/", &ptr[2]);
-				if (update_target(abs, ptr[0]))
-					rebuild = true;
-
-				free(abs);
-			} else {
-				if (update_target(&ptr[2], ptr[0]))
-					rebuild = true;
-			}
-			ptr = &buf[i+1];
+	for (size_t i = 0; i < ctx.fields_count; ++i)
+		free(ctx.fields[i]);
+
+	dsv_free(&ctx);
+	fclose(fp);
+
+
+	char *prereq_path = concat(2, dep->path, ".prereq");
+	fp = fopen(prereq_path, "rb");
+	free(prereq_path);
+	if (!fp) {
+		if (errno == ENOENT)
+			return 0;
+		else
+			fatal("redo: failed to open %s", dep->path);
+	}
+
+	dsv_init(&ctx, 2);
+
+	while (!dsv_parse_file(&ctx, fp)) {
+		char *target, *abs = NULL;
+		if (!is_absolute(ctx.fields[1])) {
+			abs = concat(3, getenv("REDO_ROOT"), "/", ctx.fields[1]);
+			target = abs;
+		} else {
+			target = ctx.fields[1];
 		}
 
-		if (read && buf[read-1] != '\n') {
-			if (buf != ptr)
-				memmove(buf, ptr, buf-ptr + sizeof buf);
-			else
-				die("redo: dependency record contains insanely long paths\n");
+		if (update_target(target, ctx.fields[0][0])) {
+			retval = build_target(dep);
+			free(abs);
+			goto exit;
 		}
+
+		free(abs);
+		free(ctx.fields[0]);
+		free(ctx.fields[1]);
 	}
 
+	goto exit2;
+
+exit:
+	for (size_t i = 0; i < ctx.fields_count; ++i)
+		free(ctx.fields[i]);
+exit2:
+	dsv_free(&ctx);
 	fclose(fp);
-	if (rebuild)
-		return build_target(dep);
 
-	return 0;
+	return retval;
 }
diff --git a/src/build.h b/src/build.h
index 9735080..2611479 100644
--- a/src/build.h
+++ b/src/build.h
@@ -11,7 +11,7 @@
 
 #include <stdbool.h>
 
-extern void add_dep(const char *target, const char *parent, int ident);
+extern void add_prereq(const char *target, const char *parent, int ident);
 extern int update_target(const char *target, int ident);
 
 #endif
diff --git a/src/redo.c b/src/redo.c
index 3927326..00da7eb 100644
--- a/src/redo.c
+++ b/src/redo.c
@@ -1,6 +1,6 @@
 /* redo.c
  *
- * Copyright (c) 2014 Tharre
+ * Copyright (c) 2014-2016 Tharre
  *
  * This software may be modified and distributed under the terms
  * of the MIT license.  See the LICENSE file for details.
@@ -79,7 +79,7 @@ int main(int argc, char *argv[]) {
 			die("%s must be called inside a .do script\n", argv[0]);
 
 		if (ident == 'a')
-			add_dep(parent, parent, ident);
+			add_prereq(parent, parent, ident);
 		else
 			for (int i = 1; i < argc; ++i) {
 				do {
@@ -87,7 +87,7 @@ int main(int argc, char *argv[]) {
 				} while (!*temp);
 
 				update_target(*temp, ident);
-				add_dep(*temp, xbasename(parent), ident);
+				add_prereq(*temp, xbasename(parent), ident);
 
 				*temp = NULL;
 			}