When encrypting, use temporary file if file gets too big

commands.cpp

@@ -17,24 +17,43 @@ void clean (const char* keyfile)
 	keys_t		keys;
 	load_keys(keyfile, &keys);
 
-	// First read the entire file into a buffer (TODO: if the buffer gets big, use a temp file instead)
-	std::string	file_contents;
+	// Read the entire file
+
+	hmac_sha1_state	hmac(keys.hmac, HMAC_KEY_LEN);	// Calculate the file's SHA1 HMAC as we go
+	uint64_t	file_size = 0;	// Keep track of the length, make sure it doesn't get too big
+	std::string	file_contents;	// First 8MB or so of the file go here
+	std::fstream	temp_file;	// The rest of the file spills into a temporary file on disk
+	temp_file.exceptions(std::fstream::badbit);
+
 	char		buffer[1024];
-	while (std::cin) {
+
+	while (std::cin && file_size < MAX_CRYPT_BYTES) {
 		std::cin.read(buffer, sizeof(buffer));
-		file_contents.append(buffer, std::cin.gcount());
+
+		size_t	bytes_read = std::cin.gcount();
+
+		hmac.add(reinterpret_cast<unsigned char*>(buffer), bytes_read);
+		file_size += bytes_read;
+
+		if (file_size <= 8388608) {
+			file_contents.append(buffer, bytes_read);
+		} else {
+			if (!temp_file.is_open()) {
+				open_tempfile(temp_file, std::fstream::in | std::fstream::out | std::fstream::binary | std::fstream::app);
+			}
+			temp_file.write(buffer, bytes_read);
+		}
 	}
-	const uint8_t*	file_data = reinterpret_cast<const uint8_t*>(file_contents.data());
-	size_t		file_len = file_contents.size();
 
 	// Make sure the file isn't so large we'll overflow the counter value (which would doom security)
-	if (file_len > MAX_CRYPT_BYTES) {
+	if (file_size >= MAX_CRYPT_BYTES) {
 		std::clog << "File too long to encrypt securely\n";
 		std::exit(1);
 	}
 
-	// Compute an HMAC of the file to use as the encryption nonce (IV) for CTR
-	// mode.  By using a hash of the file we ensure that the encryption is
+
+	// We use an HMAC of the file as the encryption nonce (IV) for CTR mode.
+	// By using a hash of the file we ensure that the encryption is
 	// deterministic so git doesn't think the file has changed when it really
 	// hasn't.  CTR mode with a synthetic IV is provably semantically secure
 	// under deterministic CPA as long as the synthetic IV is derived from a
@@ -53,19 +72,37 @@ void clean (const char* keyfile)
 	// To prevent an attacker from building a dictionary of hash values and then
 	// looking up the nonce (which must be stored in the clear to allow for
 	// decryption), we use an HMAC as opposed to a straight hash.
-	uint8_t		digest[12];
-	hmac_sha1_96(digest, file_data, file_len, keys.hmac, HMAC_KEY_LEN);
+
+	uint8_t		digest[SHA1_LEN];
+	hmac.get(digest);
 
 	// Write a header that...
 	std::cout.write("\0GITCRYPT\0", 10); // ...identifies this as an encrypted file
-	std::cout.write(reinterpret_cast<char*>(digest), 12); // ...includes the nonce
+	std::cout.write(reinterpret_cast<char*>(digest), NONCE_LEN); // ...includes the nonce
 
 	// Now encrypt the file and write to stdout
-	aes_ctr_state	state(digest, 12);
-	for (size_t i = 0; i < file_len; i += sizeof(buffer)) {
-		size_t	block_len = std::min(sizeof(buffer), file_len - i);
-		state.process_block(&keys.enc, file_data + i, reinterpret_cast<uint8_t*>(buffer), block_len);
-		std::cout.write(buffer, block_len);
+	aes_ctr_state	state(digest, NONCE_LEN);
+
+	// First read from the in-memory copy
+	const uint8_t*	file_data = reinterpret_cast<const uint8_t*>(file_contents.data());
+	size_t		file_data_len = file_contents.size();
+	for (size_t i = 0; i < file_data_len; i += sizeof(buffer)) {
+		size_t	buffer_len = std::min(sizeof(buffer), file_data_len - i);
+		state.process(&keys.enc, file_data + i, reinterpret_cast<uint8_t*>(buffer), buffer_len);
+		std::cout.write(buffer, buffer_len);
+	}
+
+	// Then read from the temporary file if applicable
+	if (temp_file.is_open()) {
+		temp_file.seekg(0);
+		while (temp_file) {
+			temp_file.read(buffer, sizeof(buffer));
+
+			size_t buffer_len = temp_file.gcount();
+
+			state.process(&keys.enc, reinterpret_cast<uint8_t*>(buffer), reinterpret_cast<uint8_t*>(buffer), buffer_len);
+			std::cout.write(buffer, buffer_len);
+		}
 	}
 }
 
@@ -97,6 +134,7 @@ void diff (const char* keyfile, const char* filename)
 		perror(filename);
 		std::exit(1);
 	}
+	in.exceptions(std::fstream::badbit);
 
 	// Read the header to get the nonce and determine if it's actually encrypted
 	char		header[22];

crypto.cpp

@@ -43,7 +43,7 @@ aes_ctr_state::aes_ctr_state (const uint8_t* arg_nonce, size_t arg_nonce_len)
 	memset(otp, '\0', sizeof(otp));
 }
 
-void aes_ctr_state::process_block (const AES_KEY* key, const uint8_t* in, uint8_t* out, size_t len)
+void aes_ctr_state::process (const AES_KEY* key, const uint8_t* in, uint8_t* out, size_t len)
 {
 	for (size_t i = 0; i < len; ++i) {
 		if (byte_counter % 16 == 0) {
@@ -63,14 +63,28 @@ void aes_ctr_state::process_block (const AES_KEY* key, const uint8_t* in, uint8_
 	}
 }
 
-// Compute HMAC-SHA1-96 (i.e. first 96 bits of HMAC-SHA1) for the given buffer with the given key
-void hmac_sha1_96 (uint8_t* out, const uint8_t* buffer, size_t buffer_len, const uint8_t* key, size_t key_len)
+hmac_sha1_state::hmac_sha1_state (const uint8_t* key, size_t key_len)
 {
-	uint8_t	full_digest[20];
-	HMAC(EVP_sha1(), key, key_len, buffer, buffer_len, full_digest, NULL);
-	memcpy(out, full_digest, 12); // Truncate to first 96 bits
+	HMAC_Init(&ctx, key, key_len, EVP_sha1());
 }
 
+hmac_sha1_state::~hmac_sha1_state ()
+{
+	HMAC_cleanup(&ctx);
+}
+
+void hmac_sha1_state::add (const uint8_t* buffer, size_t buffer_len)
+{
+	HMAC_Update(&ctx, buffer, buffer_len);
+}
+
+void hmac_sha1_state::get (uint8_t* digest)
+{
+	unsigned int len;
+	HMAC_Final(&ctx, digest, &len);
+}
+
+
 // Encrypt/decrypt an entire input stream, writing to the given output stream
 void process_stream (std::istream& in, std::ostream& out, const AES_KEY* enc_key, const uint8_t* nonce)
 {
@@ -79,7 +93,7 @@ void process_stream (std::istream& in, std::ostream& out, const AES_KEY* enc_key
 	uint8_t		buffer[1024];
 	while (in) {
 		in.read(reinterpret_cast<char*>(buffer), sizeof(buffer));
-		state.process_block(enc_key, buffer, buffer, in.gcount());
+		state.process(enc_key, buffer, buffer, in.gcount());
 		out.write(reinterpret_cast<char*>(buffer), in.gcount());
 	}
 }

crypto.hpp

@@ -2,11 +2,14 @@
 #define _CRYPTO_H
 
 #include <openssl/aes.h>
+#include <openssl/hmac.h>
 #include <stdint.h>
 #include <stddef.h>
 #include <iosfwd>
 
 enum {
+	SHA1_LEN = 20,
+	NONCE_LEN = 12,
 	HMAC_KEY_LEN = 64,
 	AES_KEY_BITS = 256,
 	MAX_CRYPT_BYTES = (1ULL<<32)*16	// Don't encrypt more than this or the CTR value will repeat itself
@@ -19,18 +22,29 @@ struct keys_t {
 void load_keys (const char* filepath, keys_t* keys);
 
 class aes_ctr_state {
-	char		nonce[12];	// First 96 bits of counter
+	char		nonce[NONCE_LEN];// First 96 bits of counter
 	uint32_t	byte_counter;	// How many bytes processed so far?
 	uint8_t		otp[16];	// The current OTP that's in use
 
 public:
 	aes_ctr_state (const uint8_t* arg_nonce, size_t arg_nonce_len);
 
-	void process_block (const AES_KEY* key, const uint8_t* in, uint8_t* out, size_t len);
+	void process (const AES_KEY* key, const uint8_t* in, uint8_t* out, size_t len);
 };
 
-// Compute HMAC-SHA1-96 (i.e. first 96 bits of HMAC-SHA1) for the given buffer with the given key
-void hmac_sha1_96 (uint8_t* out, const uint8_t* buffer, size_t buffer_len, const uint8_t* key, size_t key_len);
+class hmac_sha1_state {
+	HMAC_CTX	ctx;
+
+	// disallow copy/assignment:
+	hmac_sha1_state (const hmac_sha1_state&) { }
+	hmac_sha1_state& operator= (const hmac_sha1_state&) { return *this; }
+public:
+	hmac_sha1_state (const uint8_t* key, size_t key_len);
+	~hmac_sha1_state ();
+
+	void add (const uint8_t* buffer, size_t buffer_len);
+	void get (uint8_t*);
+};
 
 // Encrypt/decrypt an entire input stream, writing to the given output stream
 void process_stream (std::istream& in, std::ostream& out, const AES_KEY* enc_key, const uint8_t* nonce);

git-crypt.cpp

@@ -19,11 +19,14 @@ static void print_usage (const char* argv0)
 
 
 int main (int argc, const char** argv)
-{
+try {
 	// The following two lines are essential for achieving good performance:
 	std::ios_base::sync_with_stdio(false);
 	std::cin.tie(0);
 
+	std::cin.exceptions(std::ios_base::badbit);
+	std::cout.exceptions(std::ios_base::badbit);
+
 	if (argc < 3) {
 		print_usage(argv[0]);
 		return 2;
@@ -46,6 +49,8 @@ int main (int argc, const char** argv)
 	}
 
 	return 0;
+} catch (const std::ios_base::failure& e) {
+	std::cerr << "git-crypt: I/O error: " << e.what() << std::endl;
 }
 
 

util.cpp

@@ -7,6 +7,7 @@
 #include <sys/wait.h>
 #include <unistd.h>
 #include <errno.h>
+#include <fstream>
 
 int exec_command (const char* command, std::string& output)
 {
@@ -49,3 +50,32 @@ std::string resolve_path (const char* path)
 	return resolved_path;
 }
 
+void	open_tempfile (std::fstream& file, std::ios_base::openmode mode)
+{
+	const char*	tmpdir = getenv("TMPDIR");
+	size_t		tmpdir_len;
+	if (tmpdir) {
+		tmpdir_len = strlen(tmpdir);
+	} else {
+		tmpdir = "/tmp";
+		tmpdir_len = 4;
+	}
+	char*		path = new char[tmpdir_len + 18];
+	strcpy(path, tmpdir);
+	strcpy(path + tmpdir_len, "/git-crypt.XXXXXX");
+	int		fd = mkstemp(path);
+	if (fd == -1) {
+		perror("mkstemp");
+		std::exit(9);
+	}
+	file.open(path, mode);
+	if (!file.is_open()) {
+		perror("open");
+		unlink(path);
+		std::exit(9);
+	}
+	unlink(path);
+	close(fd);
+	delete[] path;
+}
+

util.hpp

@@ -2,9 +2,12 @@
 #define _UTIL_H
 
 #include <string>
+#include <ios>
+#include <iosfwd>
 
 int		exec_command (const char* command, std::string& output);
 std::string	resolve_path (const char* path);
+void		open_tempfile (std::fstream&, std::ios_base::openmode);
 
 #endif