1 files changed, 132 insertions, 0 deletions
diff --git a/src/lib9/tokenize.c b/src/lib9/tokenize.c
new file mode 100644
index 000000000..6b8c0e778
--- /dev/null
+++ b/src/lib9/tokenize.c
@@ -0,0 +1,132 @@
+/*
+Inferno lib9/tokenize.c
+http://code.google.com/p/inferno-os/source/browse/lib9/tokenize.c
+
+	Copyright © 1994-1999 Lucent Technologies Inc.  All rights reserved.
+	Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com).  All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include <lib9.h>
+
+static char qsep[] = " \t\r\n";
+
+static char*
+qtoken(char *s, char *sep)
+{
+	int quoting;
+	char *t;
+
+	quoting = 0;
+	t = s;	/* s is output string, t is input string */
+	while(*t!='\0' && (quoting || utfrune(sep, *t)==nil)){
+		if(*t != '\''){
+			*s++ = *t++;
+			continue;
+		}
+		/* *t is a quote */
+		if(!quoting){
+			quoting = 1;
+			t++;
+			continue;
+		}
+		/* quoting and we're on a quote */
+		if(t[1] != '\''){
+			/* end of quoted section; absorb closing quote */
+			t++;
+			quoting = 0;
+			continue;
+		}
+		/* doubled quote; fold one quote into two */
+		t++;
+		*s++ = *t++;
+	}
+	if(*s != '\0'){
+		*s = '\0';
+		if(t == s)
+			t++;
+	}
+	return t;
+}
+
+static char*
+etoken(char *t, char *sep)
+{
+	int quoting;
+
+	/* move to end of next token */
+	quoting = 0;
+	while(*t!='\0' && (quoting || utfrune(sep, *t)==nil)){
+		if(*t != '\''){
+			t++;
+			continue;
+		}
+		/* *t is a quote */
+		if(!quoting){
+			quoting = 1;
+			t++;
+			continue;
+		}
+		/* quoting and we're on a quote */
+		if(t[1] != '\''){
+			/* end of quoted section; absorb closing quote */
+			t++;
+			quoting = 0;
+			continue;
+		}
+		/* doubled quote; fold one quote into two */
+		t += 2;
+	}
+	return t;
+}
+
+int
+gettokens(char *s, char **args, int maxargs, char *sep)
+{
+	int nargs;
+
+	for(nargs=0; nargs<maxargs; nargs++){
+		while(*s!='\0' && utfrune(sep, *s)!=nil)
+			*s++ = '\0';
+		if(*s == '\0')
+			break;
+		args[nargs] = s;
+		s = etoken(s, sep);
+	}
+
+	return nargs;
+}
+
+int
+tokenize(char *s, char **args, int maxargs)
+{
+	int nargs;
+
+	for(nargs=0; nargs<maxargs; nargs++){
+		while(*s!='\0' && utfrune(qsep, *s)!=nil)
+			s++;
+		if(*s == '\0')
+			break;
+		args[nargs] = s;
+		s = qtoken(s, qsep);
+	}
+
+	return nargs;
+}