Re: OpenSSL breaks factor(6)

Here's a final attempt at fixing and documenting FreeBSD's factor(6).
Do what you want with the patch.  With and without OpenSSL, one now
gets

% factor +123 123 123zzzz 123zabc 123abc +123abc 0x123abc +0x123abc
123: 3 41
123: 3 41
123: 3 41
123: 3 41
1194684: 2 2 3 29 3433
1194684: 2 2 3 29 3433
1194684: 2 2 3 29 3433
1194684: 2 2 3 29 3433

* usr.bin/factor/factor.6:
  . Update documentation to note that hexadecimal strings are accepted.
  . Document that a hexadecimal number can have an optional 0x or 0X prefix.
  . Document that a 0 value in interactive mode terminates factor(6).
  . Fix the maximum value for 'stop' in primes(6).
  . While here, spell "white-space" as "whitespace" and "non-digit" as
    "nondigit".

* usr.bin/factor/factor.c:
  . Include stdbool to get acces to bool type.
  . Use consistent style for function prototypes.
  . New function. is_hex_str() looks for the longest substring and 
    determines if it is a hexadecimal number.
  . New function.  Factor (pun intended) out common code into convert_str2bn().
  . For the WIHTOUT_OPENSSL case, make BN_dec2bn() and BN_hex2bn() return 0
    on error like their OpenSSL counterparts.

* usr.bin/primes/primes.c:
  . Fix comment.
 
Index: usr.bin/factor/factor.6
===================================================================
--- usr.bin/factor/factor.6	(revision 355983)
+++ usr.bin/factor/factor.6	(working copy)
_at__at_ -36,7 +36,7 _at__at_
 .\"
 .\"   chongo <for a good prime call: 391581 * 2^216193 - 1> /\oo/\
 .\"
-.Dd October 10, 2002
+.Dd December 29, 2019
 .Dt FACTOR 6
 .Os
 .Sh NAME
_at__at_ -67,11 +67,22 _at__at_
 .Nm
 is invoked with no arguments,
 .Nm
-reads numbers, one per line, from standard input, until end of file or error.
+reads numbers, one per line, from standard input until end of file or 0
+is entered or an error occurs.
 Leading white-space and empty lines are ignored.
+.Pp
 Numbers may be preceded by a single
 .Ql + .
+Numbers can be either decimal or hexadecimal strings where the longest
+leading substring is used.
 Numbers are terminated by a non-digit character (such as a newline).
+If the string contains only decimal digits, it is treated as a
+decimal representation for a number.
+A hexadecimal string can contain an optional
+.Em 0x
+or
+.Em 0X
+prefix.
 After a number is read, it is factored.
 .Pp
 The
_at__at_ -89,7 +100,7 _at__at_
 value must not be greater than the maximum.
 The default and maximum value of
 .Ar stop
-is 3825123056546413050.
+is 18446744073709551615.
 .Pp
 When the
 .Nm primes
Index: usr.bin/factor/factor.c
===================================================================
--- usr.bin/factor/factor.c	(revision 355983)
+++ usr.bin/factor/factor.c	(working copy)
_at__at_ -71,6 +71,7 _at__at_
 #include <errno.h>
 #include <inttypes.h>
 #include <limits.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
_at__at_ -97,8 +98,8 _at__at_
 #define BN_is_one(v)		(*(v) == 1)
 #define BN_mod_word(a, b)	(*(a) % (b))
 
-static int	BN_dec2bn(BIGNUM **a, const char *str);
-static int	BN_hex2bn(BIGNUM **a, const char *str);
+static int	BN_dec2bn(BIGNUM **, const char *);
+static int	BN_hex2bn(BIGNUM **, const char *);
 static BN_ULONG BN_div_word(BIGNUM *, BN_ULONG);
 static void	BN_print_fp(FILE *, const BIGNUM *);
 
_at__at_ -105,7 +106,8 _at__at_
 #endif
 
 static void	BN_print_dec_fp(FILE *, const BIGNUM *);
-
+static void	convert_str2bn(BIGNUM **, char *);
+static bool	is_hex_str(char *);
 static void	pr_fact(BIGNUM *);	/* print factors of a value */
 static void	pr_print(BIGNUM *);	/* print a prime */
 static void	usage(void);
_at__at_ -148,21 +150,13 _at__at_
 			for (p = buf; isblank(*p); ++p);
 			if (*p == '\n' || *p == '\0')
 				continue;
-			if (*p == '-')
-				errx(1, "negative numbers aren't permitted.");
-			if (BN_dec2bn(&val, buf) == 0 &&
-			    BN_hex2bn(&val, buf) == 0)
-				errx(1, "%s: illegal numeric format.", buf);
+			convert_str2bn(&val, p);
 			pr_fact(val);
 		}
 	/* Factor the arguments. */
 	else
-		for (; *argv != NULL; ++argv) {
-			if (argv[0][0] == '-')
-				errx(1, "negative numbers aren't permitted.");
-			if (BN_dec2bn(&val, argv[0]) == 0 &&
-			    BN_hex2bn(&val, argv[0]) == 0)
-				errx(1, "%s: illegal numeric format.", argv[0]);
+		for (p = *argv; p != NULL; p = *++argv) {
+			convert_str2bn(&val, p);
 			pr_fact(val);
 		}
 	exit(0);
_at__at_ -346,7 +340,7 _at__at_
 
 	errno = 0;
 	**a = strtoul(str, &p, 10);
-	return (errno == 0 && (*p == '\n' || *p == '\0'));
+	return (errno == 0 ? 1 : 0);	/* OpenSSL returns 0 on error! */
 }
 
 static int
_at__at_ -356,7 +350,7 _at__at_
 
 	errno = 0;
 	**a = strtoul(str, &p, 16);
-	return (errno == 0 && (*p == '\n' || *p == '\0'));
+	return (errno == 0 ? 1 : 0);	/* OpenSSL returns 0 on error! */
 }
 
 static BN_ULONG
_at__at_ -370,3 +364,46 _at__at_
 }
 
 #endif
+
+/*
+ * Scan the string from left-to-right to see if the longest substring
+ * is a valid hexadecimal number.
+ */
+static bool
+is_hex_str(char *str)
+{
+	char c, *p;
+	bool saw_hex = false;
+
+	for (p = str; *p; p++) {
+		if (isdigit(*p))
+			continue;
+		c = tolower(*p);
+		if (c >= 'a' && c <= 'f') {
+			saw_hex = true;
+			continue;
+		}
+		break;	/* Not a hexadecimal digit. */
+	}
+	return saw_hex;
+}
+
+/* Convert string pointed to by *str to a bignum.  */
+static void
+convert_str2bn(BIGNUM **val, char *p)
+{
+	int n = 0;
+
+	if (*p == '+') p++;
+	if (*p == '-')
+		errx(1, "negative numbers aren't permitted.");
+	if (*p == '0') {
+		p++;
+		if (*p == 'x' || *p == 'X')
+			n = BN_hex2bn(val, ++p);
+	} else {
+		n = is_hex_str(p) ? BN_hex2bn(val, p) : BN_dec2bn(val, p);
+	}
+	if (n == 0)
+		errx(1, "%s: illegal numeric format.", p);
+}
Index: usr.bin/primes/primes.c
===================================================================
--- usr.bin/primes/primes.c	(revision 355983)
+++ usr.bin/primes/primes.c	(working copy)
_at__at_ -55,7 +55,7 _at__at_
  *	primes [-h] [start [stop]]
  *
  *	Print primes >= start and < stop.  If stop is omitted,
- *	the value 4294967295 (2^32-1) is assumed.  If start is
+ *	the value 18446744073709551615 (2^64-1) is assumed.  If start is
  *	omitted, start is read from standard input.
  *
  * validation check: there are 664579 primes between 0 and 10^7

-- 
Steve