[hexxy] DEV branch - extremely speed up hex encoding

color.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"bytes"
+	"strconv"
+	"unsafe"
+)
+
+const GREY = "\x1b[38;2;111;111;111m"
+const CLR = "\x1b[0m"
+
+var ESC = []byte{0x5c, 0x78, 0x31, 0x62, 0x5b}
+var CLEAR = []byte{0x5c, 0x78, 0x31, 0x62, 0x5b, 0x30, 0x6d}
+
+type Color struct {
+	disable bool
+	values  [256]string
+	bvalues [256][]byte
+	cvalues map[byte][]byte
+}
+
+func (c *Color) Compute() {
+	const WHITEB = "\x1b[1;37m"
+	for i := 0; i < 256; i++ {
+		var fg, bg string
+
+		lowVis := i == 0 || (i >= 16 && i <= 20) || (i >= 232 && i <= 242)
+
+		if lowVis {
+			fg = WHITEB + "\x1b[38;5;" + "255" + "m"
+			bg = "\x1b[48;5;" + strconv.Itoa(int(i)) + "m"
+		} else {
+			fg = "\x1b[38;5;" + strconv.Itoa(int(i)) + "m"
+			bg = ""
+		}
+		c.values[i] = bg + fg
+	}
+}
+
+func (c *Color) Colorize(s string, clr byte) string {
+	const NOCOLOR = "\x1b[0m"
+	return c.values[clr] + s + NOCOLOR
+}
+
+func (c *Color) ColorizeBytes(s string, byteColor []byte) []byte {
+	const NOCOLOR = "\x1b[0m"
+	b := ByteArrayToInt(byteColor)
+	return []byte(c.values[b] + s + NOCOLOR)
+}
+
+func (c *Color) ComputeBytes() {
+	const WHITEB = "\x1b[1;37m"
+	for i := 0; i < 256; i++ {
+		var fg, bg string
+		b := byte(i)
+
+		lowVis := i == 0 || (i >= 16 && i <= 20) || (i >= 232 && i <= 242)
+
+		if lowVis {
+			fg = WHITEB + "\x1b[38;5;" + "255" + "m"
+			bg = "\x1b[48;5;" + strconv.Itoa(int(i)) + "m"
+		} else {
+			fg = "\x1b[38;5;" + strconv.Itoa(int(i)) + "m"
+			bg = ""
+		}
+
+		c.values[i] = bg + fg
+		c.cvalues[b] = []byte(bg + fg)
+	}
+}
+
+func (c *Color) xComputeBytes() {
+	const Marker = '\x1b'
+	var b bytes.Buffer
+
+	for i := 0; i < 256; i++ {
+		// var fg, bg []byte
+		b.Write(ESC)
+
+		// x := string(i)
+		// y := []byte(x)
+
+		lowVis := i == 0 || (i >= 16 && i <= 20) || (i >= 232 && i <= 242)
+		if lowVis {
+			b.Write([]byte{'[', '1', ';', '3', '7', 'm'})
+			b.Write(ESC)
+			b.Write([]byte{'[', '4', '8', ';', '5'})
+			bg := make([]byte, 3)
+			bg = IntToByteArray(i)
+			b.Write(bg)
+			b.WriteByte('m')
+		} else {
+			b.Write([]byte{'[', '3', '8', ';', '5'})
+			fg := make([]byte, 3)
+			fg = IntToByteArray(i)
+			b.Write(fg)
+			b.WriteByte('m')
+		}
+		// c.values[i] = bg + fg
+		// c.bvalues[i] = bytes.Join([]byte(bg), []byte(fg))
+		c.bvalues[i] = b.Bytes()
+	}
+}
+
+func IntToByteArray(num int) []byte {
+	size := int(unsafe.Sizeof(num))
+	arr := make([]byte, size)
+	for i := 0; i < size; i++ {
+		byt := *(*uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(&num)) + uintptr(i)))
+		arr[i] = byt
+	}
+	return arr
+}
+
+func ByteArrayToInt(arr []byte) int64 {
+	val := int64(0)
+	size := len(arr)
+	for i := 0; i < size; i++ {
+		*(*uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(&val)) + uintptr(i))) = arr[i]
+	}
+	return val
+}

encode.go

@@ -2,6 +2,19 @@ package main
 
 import ()
 
+func binaryEncode(dst, src []byte) {
+	d := uint(0)
+	_, _ = src[0], dst[7]
+	for i := 7; i >= 0; i-- {
+		if src[0]&(1<<d) == 0 {
+			dst[i] = 0
+		} else {
+			dst[i] = 1
+		}
+		d++
+	}
+}
+
 // returns -1 on success
 // returns k > -1 if space found where k is index of space byte
 func binaryDecode(dst, src []byte) int {
@@ -98,6 +111,16 @@ func empty(b *[]byte) bool {
 	return true
 }
 
+// check if filename character contains problematic characters
+func isSpecial(b byte) bool {
+	switch b {
+	case '/', '!', '#', '$', '%', '^', '&', '*', '(', ')', ';', ':', '|', '{', '}', '\\', '~', '`':
+		return true
+	default:
+		return false
+	}
+}
+
 // quick binary tree check
 // probably horribly written idk it's late at night
 func parseSpecifier(b string) float64 {

hexxy.go

@@ -21,7 +21,7 @@ var opts struct {
 	Seek         int64  `short:"s" long:"seek" description:"start at <seek> bytes"`
 	Len          int64  `short:"l" long:"len" description:"stop after <len> octets"`
 	Columns      int    `short:"c" long:"columns" description:"column count"`
-	GroupSize    int    `short:"g" long:"groups" description:"group count"`
+	GroupSize    int    `short:"g" long:"groups" description:"group size of bytes"`
 	Plain        bool   `short:"p" long:"plain" description:"plain output without ascii table and offset row [often used with hexxy -r]"`
 	Upper        bool   `short:"u" long:"upper" description:"output hex in UPPERCASE format"`
 	CInclude     bool   `short:"i" long:"include" description:"output in C include format"`
@@ -64,50 +64,6 @@ var (
 	bar          = []byte("|")
 )
 
-func binaryEncode(dst, src []byte) {
-	d := uint(0)
-	_, _ = src[0], dst[7]
-	for i := 7; i >= 0; i-- {
-		if src[0]&(1<<d) == 0 {
-			dst[i] = 0
-		} else {
-			dst[i] = 1
-		}
-		d++
-	}
-}
-
-const GREY = "\x1b[38;2;111;111;111m"
-const CLR = "\x1b[0m"
-
-type Color struct {
-	disable bool
-	values  [256]string
-}
-
-func (c *Color) Compute() {
-	const WHITEB = "\x1b[1;37m"
-	for i := 0; i < 256; i++ {
-		var fg, bg string
-
-		lowVis := i == 0 || (i >= 16 && i <= 20) || (i >= 232 && i <= 242)
-
-		if lowVis {
-			fg = WHITEB + "\x1b[38;5;" + "255" + "m"
-			bg = "\x1b[48;5;" + strconv.Itoa(int(i)) + "m"
-		} else {
-			fg = "\x1b[38;5;" + strconv.Itoa(int(i)) + "m"
-			bg = ""
-		}
-		c.values[i] = bg + fg
-	}
-}
-
-func (c *Color) Colorize(s string, clr byte) string {
-	const NOCOLOR = "\x1b[0m"
-	return c.values[clr] + s + NOCOLOR
-}
-
 func inputIsPipe() bool {
 	stat, _ := os.Stdin.Stat()
 	return stat.Mode()&os.ModeCharDevice != os.ModeCharDevice
@@ -118,53 +74,7 @@ func outputIsPipe() bool {
 	return stat.Mode()&os.ModeCharDevice != os.ModeCharDevice
 }
 
-func HexdumpPlain(file *os.File) error {
-	var i uint64
-	reader := bufio.NewReaderSize(file, 10*1024)
-
-	for {
-		b, err := reader.ReadByte()
-		if errors.Is(err, io.EOF) {
-			break
-		}
-		if err != nil {
-			return fmt.Errorf("Failed to read %v: %w\n", file.Name(), err)
-		}
-
-		if i%30 == 0 {
-			println()
-		}
-
-		fmt.Printf("%02x", b)
-		i++
-	}
-
-	return nil
-}
-
-// func plain2Binary(file *os.File) error {
-// 	return reverse(os.Stdout, os.Stdin)
-// }
-
-// func getOffsetFormat() error {
-// 	var prefix string
-// 	var suffix string
-// 	var format string
-
-// 	switch opts.OffsetFormat {
-// 	case "d":
-// 		format = prefix + "%08d  " + suffix
-// 	case "o":
-// 		format = prefix + "%08o  " + suffix
-// 	case "x":
-// 		format = prefix + "%08x  " + suffix
-// 	default:
-// 		return fmt.Errorf("Offset format must be [d|o|x]")
-// 	}
-// 	return nil
-// }
-
-func XXD(r io.Reader, w io.Writer, filename string) error {
+func XXD(r io.Reader, w io.Writer, filename string, color *Color) error {
 	var (
 		lineOffset  int64
 		hexOffset   = make([]byte, 6)
@@ -182,10 +92,10 @@ func XXD(r io.Reader, w io.Writer, filename string) error {
 
 	if dumpType == dumpCformat {
 		_ = copy(varDeclChar[0:14], unsignedChar[:])
-		_ = copy(varDeclInt[0:14], lenEquals[:])
+		_ = copy(varDeclInt[0:16], unsignedInt[:])
 
 		for i := 0; i < len(filename); i++ {
-			if filename[i] != '.' {
+			if !isSpecial(filename[i]) {
 				varDeclChar[14+i] = filename[i]
 				varDeclInt[16+i] = filename[i]
 			} else {
@@ -261,7 +171,7 @@ func XXD(r io.Reader, w io.Writer, filename string) error {
 
 	for {
 		n, err = io.ReadFull(r, line)
-		if err != nil && errors.Is(err, io.EOF) && errors.Is(err, io.ErrUnexpectedEOF) {
+		if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
 			return fmt.Errorf("hexxy: %v", err)
 		}
 
@@ -302,7 +212,7 @@ func XXD(r io.Reader, w io.Writer, filename string) error {
 			nulLine++
 
 			if nulLine > 1 {
-				lineOffset++
+				lineOffset++ // still increment offset while printing crunched lines with '*'
 				continue
 			}
 		}
@@ -352,6 +262,9 @@ func XXD(r io.Reader, w io.Writer, filename string) error {
 			// hex values -- default
 			for i, k := 0, octs; i < n; i, k = i+1, k+octs {
 				hexEncode(char, line[i:i+1], caps)
+
+				// s := color.Colorize(string(char), byte(i))
+				// w.Write([]byte(s))
 				w.Write(char)
 				c++
 
@@ -467,10 +380,13 @@ func Hexxy(args []string) error {
 	defer out.Flush()
 
 	if opts.Reverse {
-
+		if err := XXDReverse(infile, outfile); err != nil {
+			return fmt.Errorf("hexxy: %v", err.Error())
+		}
+		return nil
 	}
 
-	if err := XXD(infile, out, infile.Name()); err != nil {
+	if err := XXD(infile, out, infile.Name(), color); err != nil {
 		return fmt.Errorf("hexxy: %v", err.Error())
 	}
 
@@ -482,6 +398,21 @@ hexxy is a command line hex dumping tool
 
 Examples:
 	hexxy [OPTIONS] input-file
+
+	# Include a binary as a C variable
+	hexxy -i input-file > output.c
+
+	# Use plain non-formatted output
+	hexxy -p input-file
+
+	# Reverse plain non-formatted output (reverse plain)
+	hexxy -rp input-file
+
+	# Show output with a space in between N groups of bytes
+	hexxy -g1 input-file ... -> outputs: 00000000: 0f 1a ff ff 00 aa
+
+	# Seek to N bytes in an input file
+	hexxy -s 12546 input-file
 `
 
 // extra usage examples
@@ -500,6 +431,7 @@ func main() {
 	parser := flags.NewParser(&opts, flags.Default)
 	args, err := parser.Parse()
 	if flags.WroteHelp(err) {
+		fmt.Print(usage_msg)
 		os.Exit(0)
 	}
 	if err != nil {
@@ -508,12 +440,14 @@ func main() {
 
 	if !inputIsPipe() && len(args) == 0 {
 		parser.WriteHelp(os.Stderr)
+		fmt.Print(usage_msg)
 		os.Exit(0)
 	}
 
 	if opts.Verbose {
 		Debug = log.Printf
 	}
+
 	if err := Hexxy(args); err != nil {
 		log.Fatal(err)
 	}

reverse.go

@@ -2,84 +2,110 @@ package main
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"io"
-	"os"
-	"strconv"
-	"strings"
 )
 
-// func reverse(w io.Writer, path string) error {
-func reverse(w io.Writer, f *os.File) error {
-	// f, err := os.Open(path)
-	// if err != nil {
-	// 	return err
-	// }
-	// defer f.Close()
-	s := bufio.NewScanner(f)
+func XXDReverse(r io.Reader, w io.Writer) error {
+	var (
+		cols int
+		octs int
+		char = make([]byte, 1)
+	)
 
-	star := false
-	var prev uint64
-	var data []byte
-	var zero [16]byte
-
-	for s.Scan() {
-		line := s.Text()
-		if line == "*" {
-			star = true
-			continue
-		}
-
-		if len(line) < len("00000000") {
-			return fmt.Errorf("invalid line %q, missing address prefix", line)
-		}
-
-		part := line[:len("00000000")]
-		line = line[len("00000000"):]
-
-		addr, err := strconv.ParseUint(part, 16, 32)
-		if err != nil {
-			return err
-		}
-
-		if star {
-			for i := prev + 16; i < addr; i += 16 {
-				data = append(data, zero[:]...)
-			}
-			star = false
-		}
-
-		prev = addr
-		pos := strings.IndexByte(line, '|')
-
-		if pos != -1 {
-			line = line[:pos]
-		}
-
-		for len(line) > 0 {
-			line = strings.TrimSpace(line)
-			pos := strings.IndexByte(line, ' ')
-			if pos == -1 {
-				pos = len(line)
-			}
-
-			part := line[:pos]
-			line = line[pos:]
-
-			b, err := strconv.ParseUint(part, 16, 8)
-			if err != nil {
-				return err
-			}
-
-			data = append(data, byte(b))
-		}
-	}
-	if err := s.Err(); err != nil {
-		return err
+	if opts.Columns != -1 {
+		cols = opts.Columns
 	}
 
-	if _, err := w.Write(data); err != nil {
-		return err
+	switch dumpType {
+	case dumpBinary:
+		octs = 8
+	case dumpCformat:
+		octs = 4
+	default:
+		octs = 2
+	}
+
+	if opts.Len != -1 {
+		if opts.Len < int64(cols) {
+			cols = int(opts.Len)
+		}
+	}
+
+	if octs < 1 {
+		octs = cols
+	}
+
+	c := int64(0)
+	rd := bufio.NewReader(r)
+	for {
+		line, err := rd.ReadBytes('\n')
+		n := len(line)
+		if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+			return fmt.Errorf("hexxy: %v", err)
+		}
+
+		if n == 0 {
+			return nil
+		}
+
+		if dumpType == dumpHex {
+			for i := 0; n >= octs; {
+				if rv := hexDecode(char, line[i:i+octs]); rv == 0 {
+					w.Write(char)
+					i += 2
+					n -= 2
+					c++
+				} else if rv == -1 {
+					i++
+					n--
+				} else {
+					// rv == -2
+					i += 2
+					n -= 2
+				}
+			}
+		} else if dumpType == dumpBinary {
+			for i := 0; n >= octs; {
+				if binaryDecode(char, line[i:i+octs]) != -1 {
+					i++
+					n--
+					continue
+				} else {
+					w.Write(char)
+					i += 8
+					n -= 8
+					c++
+				}
+			}
+		} else if dumpType == dumpPlain {
+			for i := 0; n >= octs; i++ {
+				if hexDecode(char, line[i:i+octs]) == 0 {
+					w.Write(char)
+					c++
+				}
+				n--
+			}
+		} else if dumpType == dumpCformat {
+			for i := 0; n >= octs; {
+				if rv := hexDecode(char, line[i:i+octs]); rv == 0 {
+					w.Write(char)
+					i += 4
+					n -= 4
+					c++
+				} else if rv == -1 {
+					i++
+					n--
+				} else { // rv == -2
+					i += 2
+					n -= 2
+				}
+			}
+		}
+
+		if c == int64(cols) && cols > 0 {
+			return nil
+		}
 	}
-	return nil
 }