void test_bitmap(const char* filename) {
	std::vector<bool> res(0x100000000L, 0);
	FILE* f = fopen(filename, "r");
	if (f == NULL) {
		printf("File not open %s\n", filename);
		return;
	}
	char buf[1024];
	size_t cnt = 0, uniq_cnt = 0, uniq_total = 0;
	char* r;
	size_t size = 0;
	while (fgets(buf, 1024, f) != NULL) {
		size_t len = strlen(buf);
		cnt++;
		crc32_t crc = crc32(buf, len);
		if(!res[crc]) {
			res[crc] = true;
			uniq_total += len;
			uniq_cnt++;
		}

		if (cnt % 1000000 == 0) printf("Time %d ms. Total %d rows. Unique %d rows. Size %d bytes\r", (int)clock(), cnt, (int)uniq_cnt, (int)uniq_total);
	}
	printf("Time %d ms. Total %d rows. Unique %d rows. Size %d bytes\n", (int)clock(), cnt, (int)uniq_cnt, (int)uniq_total);
	system("pause");
	fclose(f);
}

  4 862 499 089 edit
     13 686 109 edit_area
    928 963 216 edit_artist
 28 998 579 350 edit_data
      3 422 839 edit_event
     16 323 279 edit_instrument
     34 436 585 edit_label
  4 244 196 429 edit_note
     33 788 599 edit_note_recipient
     11 592 277 edit_place
    950 615 025 edit_recording
    383 683 200 edit_release
    269 146 948 edit_release_group
      1 634 957 edit_series
     86 015 723 edit_url
    114 429 566 edit_work
    821 822 488 vote
17 файлов 41 774 835 679 байт

void test_bitmap(const char* filename, const char* filename_out) {
	FILE* f = fopen(filename, "r");
	if (f == NULL) {
		printf("Not open %s\n", filename);
		return;
	}
	FILE* f_out = fopen(filename_out, "w");
	if(f_out == NULL) {
		printf("Not open %s\n", filename_out);
		fclose(f);
		return;
	}
	printf("Find unique row in %s\n", filename);
	char buf[1024];
	size_t cnt = 0, uniq_cnt = 0, uniq_total = 0, dbl_cnt = 0;
	char* r;
	size_t size = 0;
	std::map<crc32_t, std::string> map_dbl;
	{
		std::vector<bool> dbl(0x100000000L, 0);
		{
			std::vector<bool> res(0x100000000L, 0);
#ifdef TEST
			while (fgets(buf, sizeof(buf), f) != NULL && cnt < TEST) {
#else
			while (fgets(buf, sizeof(buf), f) != NULL) {
#endif
				size_t len = strlen(buf);
				//uniq_total += len;
				crc32_t crc = crc32(buf, len);
				if(!res[crc]) { // CRC встретилось впервые
					res[crc] = true;
					uniq_total += len;
					fputs(buf, f_out);
					uniq_cnt++;
				} else if(!dbl[crc]) { // Повтор CRC, сохраняем для перепроверки
					dbl[crc] = true;
					dbl_cnt++;
				}

				cnt++;
				if ((cnt & 0xFFFFF) == 0) printf("Time %d ms. Total %d rows. Unique %d rows. Size %d kbytes\r", (int)clock(), cnt, (int)uniq_cnt, (int)(uniq_total / 1024));
			}
			printf("Time %d ms. Total %d rows. Unique %d rows. Size %d kbytes\n", (int)clock(), cnt, (int)uniq_cnt, (int)(uniq_total / 1024));
		}
		//res.clear();
		//res.resize(0);

		// Загрузка найденных строк с повторами в map_dbl
		printf("Doubles CRC %d rows. \n", (int)dbl_cnt);
		fclose(f_out);
		f_out = fopen(filename_out, "r");
		if (f_out == NULL) {
			printf("Not open %s\n", filename_out);
			fclose(f);
			return;
		}
		cnt = 0;
		while (fgets(buf, sizeof(buf), f_out) != NULL) {
			size_t len = strlen(buf);
			crc32_t crc = crc32(buf, len);
			if(dbl[crc]) {
				map_dbl[crc] = buf;
			}
			cnt++;
			if ((cnt & 0xFFFFF) == 0) printf("Time %d ms. Total %d rows.\r", (int)clock(), cnt);
		}
		printf("Time %d ms. Total %d rows.\n", (int)clock(), cnt);
		fclose(f_out);
	}
	//dbl.clear();
	//dbl.resize(0);

	// Поиск коллизий (совпало CRC)
	std::set<std::string> dop;
	fseek(f, SEEK_SET, 0);
	cnt = 0;
#ifdef TEST
	while (fgets(buf, sizeof(buf), f) != NULL && cnt < TEST) {
#else
	while (fgets(buf, sizeof(buf), f) != NULL) {
#endif
		size_t len = strlen(buf);
		crc32_t crc = crc32(buf, len);
		std::map<crc32_t, std::string>::iterator it = map_dbl.find(crc);
		if(it != map_dbl.end() && it->second != buf) {
			dop.insert(buf);
		}

		cnt++;
		if ((cnt & 0xFFFFF) == 0) printf("Time %d ms. Total %d rows. Unique %d rows.\r", (int)clock(), cnt, (int)dop.size());
	}
	printf("Time %d ms. Total %d rows. Unique %d rows.\n", (int)clock(), cnt, (int)dop.size());

	// Дозапись найденного
	if(dop.size() != 0) {
		f_out = fopen(filename_out, "a");
		if (f_out == NULL) {
			printf("Not open %s\n", filename_out);
			fclose(f);
			return;
		}
		for(std::set<std::string>::iterator it = dop.begin(); it != dop.end(); it++) {
			fputs(it->c_str(), f_out);
		}
		fclose(f_out);
	}
	fclose(f);
	printf("Time %d ms. Total unique %d rows.\n", (int)clock(), (int)dop.size() + uniq_cnt);
}