diff --git a/selfhost/tokenizer.n b/selfhost/tokenizer.n index d9b6dd6..e629001 100644 --- a/selfhost/tokenizer.n +++ b/selfhost/tokenizer.n @@ -70,6 +70,7 @@ var token_offset: std.containers.PinnedVector(u32, MAX_TOKENS) = undefined; // TODO add array initializers var token_handlers: [256]Handler = undefined; +var digit_value: [256]u8 = undefined; fn fill_handlers(start: u64, end: u64, handler: Handler) inline void { var curr_h = token_handlers[start].&; @@ -105,6 +106,29 @@ fn init() void { token_handlers['@'] = ident_h; token_handlers['_'] = ident_h; + { + var ch: u32 = 0; + loop(ch <= 0xFF) { + digit_value[ch] = 0xFF; + ch += 1; + } + ch = '0'; + loop(ch <= '9') { + digit_value[ch] = @truncate(u8, ch) - '0'; + ch += 1; + } + ch = 'a'; + loop(ch <= 'f') { + digit_value[ch] = @truncate(u8, ch) - 'a'; + ch += 1; + } + ch = 'A'; + loop(ch <= 'F') { + digit_value[ch] = @truncate(u8, ch) - 'A'; + ch += 1; + } + } + fill_handlers('0', '9' + 1, fn (context: *TokenizationContext, ch: u8) void { context.add_token(TokenType.int_literal); var base: u8 = 10; @@ -119,7 +143,7 @@ fn init() void { base = 2; } } - loop(digit_value(context.peek(0)) < base) { + loop(digit_value[context.peek(0)] < base) { context.advance(1); } }.&); @@ -171,19 +195,6 @@ fn init() void { }.&; } -fn digit_value(ch: u8) inline u8 { - if('0' <= ch && ch <= '9') { - return ch - '0' + 0x0; - } - if('a' <= ch && ch <= 'f') { - return ch - 'a' + 0xa; - } - if('A' <= ch && ch <= 'F') { - return ch - 'A' + 0xA; - } - return 0xFF; -} - fn badchar_handler(context: *TokenizationContext) void { context.report_error("Bad character".&); } diff --git a/std/containers/pinned_vector.n b/std/containers/pinned_vector.n index ef6cf95..155d605 100644 --- a/std/containers/pinned_vector.n +++ b/std/containers/pinned_vector.n @@ -23,6 +23,22 @@ const GenericVector = struct { self.committed_size = 0; } + fn deinit(self: *@This(), bytes: usize) void { + syscalls.munmap( + self.ptr, + bytes, + ); + } + + fn add(self: *@This()) u32 { + const retval = self.size; + self.size += 1; + if(self.size > self.committed_size) { + self.committed_size = self.size; + } + return retval; + } + fn madvise(self: *const @This(), offset: u64, size: u64, advice: u32) inline void { syscalls.madvise(self.ptr + offset, size, advice); } @@ -41,10 +57,7 @@ fn PinnedVector(comptime T: type, comptime capacity: u32) type { } fn deinit(self: *@This()) inline void { - syscalls.munmap( - self.gen.ptr, - @size_of(T) * capacity, - ); + self.gen.deinit(@size_of(T) * capacity); } fn append_assume_capacity(self: *@This(), value: T) u32 { @@ -52,13 +65,8 @@ fn PinnedVector(comptime T: type, comptime capacity: u32) type { return self.add(); } - fn add(self: *@This()) u32 { - const retval = self.gen.size; - self.gen.size += 1; - if(self.gen.size > self.gen.committed_size) { - self.gen.committed_size = self.gen.size; - } - return retval; + fn add(self: *@This()) inline u32 { + return self.gen.add(); } fn items_to_pages(items: u64) inline u64 {