From a769eb2024f99bcfee4530827d727484e2d8ace1 Mon Sep 17 00:00:00 2001 From: mizi Date: Thu, 1 Feb 2024 15:12:24 -0900 Subject: [PATCH] [hexxy] DEV branch - extremely speed up hex encoding --- color.go | 122 ++++++++++++++++++++++++++++++++++++++ encode.go | 23 ++++++++ hexxy.go | 130 ++++++++++------------------------------ reverse.go | 170 ++++++++++++++++++++++++++++++----------------------- 4 files changed, 275 insertions(+), 170 deletions(-) create mode 100644 color.go diff --git a/color.go b/color.go new file mode 100644 index 0000000..3994877 --- /dev/null +++ b/color.go @@ -0,0 +1,122 @@ +package main + +import ( + "bytes" + "strconv" + "unsafe" +) + +const GREY = "\x1b[38;2;111;111;111m" +const CLR = "\x1b[0m" + +var ESC = []byte{0x5c, 0x78, 0x31, 0x62, 0x5b} +var CLEAR = []byte{0x5c, 0x78, 0x31, 0x62, 0x5b, 0x30, 0x6d} + +type Color struct { + disable bool + values [256]string + bvalues [256][]byte + cvalues map[byte][]byte +} + +func (c *Color) Compute() { + const WHITEB = "\x1b[1;37m" + for i := 0; i < 256; i++ { + var fg, bg string + + lowVis := i == 0 || (i >= 16 && i <= 20) || (i >= 232 && i <= 242) + + if lowVis { + fg = WHITEB + "\x1b[38;5;" + "255" + "m" + bg = "\x1b[48;5;" + strconv.Itoa(int(i)) + "m" + } else { + fg = "\x1b[38;5;" + strconv.Itoa(int(i)) + "m" + bg = "" + } + c.values[i] = bg + fg + } +} + +func (c *Color) Colorize(s string, clr byte) string { + const NOCOLOR = "\x1b[0m" + return c.values[clr] + s + NOCOLOR +} + +func (c *Color) ColorizeBytes(s string, byteColor []byte) []byte { + const NOCOLOR = "\x1b[0m" + b := ByteArrayToInt(byteColor) + return []byte(c.values[b] + s + NOCOLOR) +} + +func (c *Color) ComputeBytes() { + const WHITEB = "\x1b[1;37m" + for i := 0; i < 256; i++ { + var fg, bg string + b := byte(i) + + lowVis := i == 0 || (i >= 16 && i <= 20) || (i >= 232 && i <= 242) + + if lowVis { + fg = WHITEB + "\x1b[38;5;" + "255" + "m" + bg = "\x1b[48;5;" + strconv.Itoa(int(i)) + "m" + } else { + fg = "\x1b[38;5;" + strconv.Itoa(int(i)) + "m" + bg = "" + } + + c.values[i] = bg + fg + c.cvalues[b] = []byte(bg + fg) + } +} + +func (c *Color) xComputeBytes() { + const Marker = '\x1b' + var b bytes.Buffer + + for i := 0; i < 256; i++ { + // var fg, bg []byte + b.Write(ESC) + + // x := string(i) + // y := []byte(x) + + lowVis := i == 0 || (i >= 16 && i <= 20) || (i >= 232 && i <= 242) + if lowVis { + b.Write([]byte{'[', '1', ';', '3', '7', 'm'}) + b.Write(ESC) + b.Write([]byte{'[', '4', '8', ';', '5'}) + bg := make([]byte, 3) + bg = IntToByteArray(i) + b.Write(bg) + b.WriteByte('m') + } else { + b.Write([]byte{'[', '3', '8', ';', '5'}) + fg := make([]byte, 3) + fg = IntToByteArray(i) + b.Write(fg) + b.WriteByte('m') + } + // c.values[i] = bg + fg + // c.bvalues[i] = bytes.Join([]byte(bg), []byte(fg)) + c.bvalues[i] = b.Bytes() + } +} + +func IntToByteArray(num int) []byte { + size := int(unsafe.Sizeof(num)) + arr := make([]byte, size) + for i := 0; i < size; i++ { + byt := *(*uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(&num)) + uintptr(i))) + arr[i] = byt + } + return arr +} + +func ByteArrayToInt(arr []byte) int64 { + val := int64(0) + size := len(arr) + for i := 0; i < size; i++ { + *(*uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(&val)) + uintptr(i))) = arr[i] + } + return val +} diff --git a/encode.go b/encode.go index 20fa5ac..91d76c5 100644 --- a/encode.go +++ b/encode.go @@ -2,6 +2,19 @@ package main import () +func binaryEncode(dst, src []byte) { + d := uint(0) + _, _ = src[0], dst[7] + for i := 7; i >= 0; i-- { + if src[0]&(1< -1 if space found where k is index of space byte func binaryDecode(dst, src []byte) int { @@ -98,6 +111,16 @@ func empty(b *[]byte) bool { return true } +// check if filename character contains problematic characters +func isSpecial(b byte) bool { + switch b { + case '/', '!', '#', '$', '%', '^', '&', '*', '(', ')', ';', ':', '|', '{', '}', '\\', '~', '`': + return true + default: + return false + } +} + // quick binary tree check // probably horribly written idk it's late at night func parseSpecifier(b string) float64 { diff --git a/hexxy.go b/hexxy.go index 06e3bab..1376909 100644 --- a/hexxy.go +++ b/hexxy.go @@ -21,7 +21,7 @@ var opts struct { Seek int64 `short:"s" long:"seek" description:"start at bytes"` Len int64 `short:"l" long:"len" description:"stop after octets"` Columns int `short:"c" long:"columns" description:"column count"` - GroupSize int `short:"g" long:"groups" description:"group count"` + GroupSize int `short:"g" long:"groups" description:"group size of bytes"` Plain bool `short:"p" long:"plain" description:"plain output without ascii table and offset row [often used with hexxy -r]"` Upper bool `short:"u" long:"upper" description:"output hex in UPPERCASE format"` CInclude bool `short:"i" long:"include" description:"output in C include format"` @@ -64,50 +64,6 @@ var ( bar = []byte("|") ) -func binaryEncode(dst, src []byte) { - d := uint(0) - _, _ = src[0], dst[7] - for i := 7; i >= 0; i-- { - if src[0]&(1<= 16 && i <= 20) || (i >= 232 && i <= 242) - - if lowVis { - fg = WHITEB + "\x1b[38;5;" + "255" + "m" - bg = "\x1b[48;5;" + strconv.Itoa(int(i)) + "m" - } else { - fg = "\x1b[38;5;" + strconv.Itoa(int(i)) + "m" - bg = "" - } - c.values[i] = bg + fg - } -} - -func (c *Color) Colorize(s string, clr byte) string { - const NOCOLOR = "\x1b[0m" - return c.values[clr] + s + NOCOLOR -} - func inputIsPipe() bool { stat, _ := os.Stdin.Stat() return stat.Mode()&os.ModeCharDevice != os.ModeCharDevice @@ -118,53 +74,7 @@ func outputIsPipe() bool { return stat.Mode()&os.ModeCharDevice != os.ModeCharDevice } -func HexdumpPlain(file *os.File) error { - var i uint64 - reader := bufio.NewReaderSize(file, 10*1024) - - for { - b, err := reader.ReadByte() - if errors.Is(err, io.EOF) { - break - } - if err != nil { - return fmt.Errorf("Failed to read %v: %w\n", file.Name(), err) - } - - if i%30 == 0 { - println() - } - - fmt.Printf("%02x", b) - i++ - } - - return nil -} - -// func plain2Binary(file *os.File) error { -// return reverse(os.Stdout, os.Stdin) -// } - -// func getOffsetFormat() error { -// var prefix string -// var suffix string -// var format string - -// switch opts.OffsetFormat { -// case "d": -// format = prefix + "%08d " + suffix -// case "o": -// format = prefix + "%08o " + suffix -// case "x": -// format = prefix + "%08x " + suffix -// default: -// return fmt.Errorf("Offset format must be [d|o|x]") -// } -// return nil -// } - -func XXD(r io.Reader, w io.Writer, filename string) error { +func XXD(r io.Reader, w io.Writer, filename string, color *Color) error { var ( lineOffset int64 hexOffset = make([]byte, 6) @@ -182,10 +92,10 @@ func XXD(r io.Reader, w io.Writer, filename string) error { if dumpType == dumpCformat { _ = copy(varDeclChar[0:14], unsignedChar[:]) - _ = copy(varDeclInt[0:14], lenEquals[:]) + _ = copy(varDeclInt[0:16], unsignedInt[:]) for i := 0; i < len(filename); i++ { - if filename[i] != '.' { + if !isSpecial(filename[i]) { varDeclChar[14+i] = filename[i] varDeclInt[16+i] = filename[i] } else { @@ -261,7 +171,7 @@ func XXD(r io.Reader, w io.Writer, filename string) error { for { n, err = io.ReadFull(r, line) - if err != nil && errors.Is(err, io.EOF) && errors.Is(err, io.ErrUnexpectedEOF) { + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { return fmt.Errorf("hexxy: %v", err) } @@ -302,7 +212,7 @@ func XXD(r io.Reader, w io.Writer, filename string) error { nulLine++ if nulLine > 1 { - lineOffset++ + lineOffset++ // still increment offset while printing crunched lines with '*' continue } } @@ -352,6 +262,9 @@ func XXD(r io.Reader, w io.Writer, filename string) error { // hex values -- default for i, k := 0, octs; i < n; i, k = i+1, k+octs { hexEncode(char, line[i:i+1], caps) + + // s := color.Colorize(string(char), byte(i)) + // w.Write([]byte(s)) w.Write(char) c++ @@ -467,10 +380,13 @@ func Hexxy(args []string) error { defer out.Flush() if opts.Reverse { - + if err := XXDReverse(infile, outfile); err != nil { + return fmt.Errorf("hexxy: %v", err.Error()) + } + return nil } - if err := XXD(infile, out, infile.Name()); err != nil { + if err := XXD(infile, out, infile.Name(), color); err != nil { return fmt.Errorf("hexxy: %v", err.Error()) } @@ -482,6 +398,21 @@ hexxy is a command line hex dumping tool Examples: hexxy [OPTIONS] input-file + + # Include a binary as a C variable + hexxy -i input-file > output.c + + # Use plain non-formatted output + hexxy -p input-file + + # Reverse plain non-formatted output (reverse plain) + hexxy -rp input-file + + # Show output with a space in between N groups of bytes + hexxy -g1 input-file ... -> outputs: 00000000: 0f 1a ff ff 00 aa + + # Seek to N bytes in an input file + hexxy -s 12546 input-file ` // extra usage examples @@ -500,6 +431,7 @@ func main() { parser := flags.NewParser(&opts, flags.Default) args, err := parser.Parse() if flags.WroteHelp(err) { + fmt.Print(usage_msg) os.Exit(0) } if err != nil { @@ -508,12 +440,14 @@ func main() { if !inputIsPipe() && len(args) == 0 { parser.WriteHelp(os.Stderr) + fmt.Print(usage_msg) os.Exit(0) } if opts.Verbose { Debug = log.Printf } + if err := Hexxy(args); err != nil { log.Fatal(err) } diff --git a/reverse.go b/reverse.go index d6f311e..e10d7fe 100644 --- a/reverse.go +++ b/reverse.go @@ -2,84 +2,110 @@ package main import ( "bufio" + "errors" "fmt" "io" - "os" - "strconv" - "strings" ) -// func reverse(w io.Writer, path string) error { -func reverse(w io.Writer, f *os.File) error { - // f, err := os.Open(path) - // if err != nil { - // return err - // } - // defer f.Close() - s := bufio.NewScanner(f) +func XXDReverse(r io.Reader, w io.Writer) error { + var ( + cols int + octs int + char = make([]byte, 1) + ) - star := false - var prev uint64 - var data []byte - var zero [16]byte - - for s.Scan() { - line := s.Text() - if line == "*" { - star = true - continue - } - - if len(line) < len("00000000") { - return fmt.Errorf("invalid line %q, missing address prefix", line) - } - - part := line[:len("00000000")] - line = line[len("00000000"):] - - addr, err := strconv.ParseUint(part, 16, 32) - if err != nil { - return err - } - - if star { - for i := prev + 16; i < addr; i += 16 { - data = append(data, zero[:]...) - } - star = false - } - - prev = addr - pos := strings.IndexByte(line, '|') - - if pos != -1 { - line = line[:pos] - } - - for len(line) > 0 { - line = strings.TrimSpace(line) - pos := strings.IndexByte(line, ' ') - if pos == -1 { - pos = len(line) - } - - part := line[:pos] - line = line[pos:] - - b, err := strconv.ParseUint(part, 16, 8) - if err != nil { - return err - } - - data = append(data, byte(b)) - } - } - if err := s.Err(); err != nil { - return err + if opts.Columns != -1 { + cols = opts.Columns } - if _, err := w.Write(data); err != nil { - return err + switch dumpType { + case dumpBinary: + octs = 8 + case dumpCformat: + octs = 4 + default: + octs = 2 + } + + if opts.Len != -1 { + if opts.Len < int64(cols) { + cols = int(opts.Len) + } + } + + if octs < 1 { + octs = cols + } + + c := int64(0) + rd := bufio.NewReader(r) + for { + line, err := rd.ReadBytes('\n') + n := len(line) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + return fmt.Errorf("hexxy: %v", err) + } + + if n == 0 { + return nil + } + + if dumpType == dumpHex { + for i := 0; n >= octs; { + if rv := hexDecode(char, line[i:i+octs]); rv == 0 { + w.Write(char) + i += 2 + n -= 2 + c++ + } else if rv == -1 { + i++ + n-- + } else { + // rv == -2 + i += 2 + n -= 2 + } + } + } else if dumpType == dumpBinary { + for i := 0; n >= octs; { + if binaryDecode(char, line[i:i+octs]) != -1 { + i++ + n-- + continue + } else { + w.Write(char) + i += 8 + n -= 8 + c++ + } + } + } else if dumpType == dumpPlain { + for i := 0; n >= octs; i++ { + if hexDecode(char, line[i:i+octs]) == 0 { + w.Write(char) + c++ + } + n-- + } + } else if dumpType == dumpCformat { + for i := 0; n >= octs; { + if rv := hexDecode(char, line[i:i+octs]); rv == 0 { + w.Write(char) + i += 4 + n -= 4 + c++ + } else if rv == -1 { + i++ + n-- + } else { // rv == -2 + i += 2 + n -= 2 + } + } + } + + if c == int64(cols) && cols > 0 { + return nil + } } - return nil }