mirror of
https://github.com/wheremyfoodat/Panda3DS.git
synced 2025-06-03 20:37:18 +12:00
Add xxhash
This commit is contained in:
parent
3606c005c5
commit
f82b27ddba
84 changed files with 66504 additions and 3 deletions
2
third_party/xxhash/tests/collisions/.gitignore
vendored
Normal file
2
third_party/xxhash/tests/collisions/.gitignore
vendored
Normal file
|
@ -0,0 +1,2 @@
|
|||
#build artefacts
|
||||
collisionsTest
|
339
third_party/xxhash/tests/collisions/LICENSE
vendored
Normal file
339
third_party/xxhash/tests/collisions/LICENSE
vendored
Normal file
|
@ -0,0 +1,339 @@
|
|||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 2, June 1991
|
||||
|
||||
Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The licenses for most software are designed to take away your
|
||||
freedom to share and change it. By contrast, the GNU General Public
|
||||
License is intended to guarantee your freedom to share and change free
|
||||
software--to make sure the software is free for all its users. This
|
||||
General Public License applies to most of the Free Software
|
||||
Foundation's software and to any other program whose authors commit to
|
||||
using it. (Some other Free Software Foundation software is covered by
|
||||
the GNU Lesser General Public License instead.) You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
this service if you wish), that you receive source code or can get it
|
||||
if you want it, that you can change the software or use pieces of it
|
||||
in new free programs; and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to make restrictions that forbid
|
||||
anyone to deny you these rights or to ask you to surrender the rights.
|
||||
These restrictions translate to certain responsibilities for you if you
|
||||
distribute copies of the software, or if you modify it.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must give the recipients all the rights that
|
||||
you have. You must make sure that they, too, receive or can get the
|
||||
source code. And you must show them these terms so they know their
|
||||
rights.
|
||||
|
||||
We protect your rights with two steps: (1) copyright the software, and
|
||||
(2) offer you this license which gives you legal permission to copy,
|
||||
distribute and/or modify the software.
|
||||
|
||||
Also, for each author's protection and ours, we want to make certain
|
||||
that everyone understands that there is no warranty for this free
|
||||
software. If the software is modified by someone else and passed on, we
|
||||
want its recipients to know that what they have is not the original, so
|
||||
that any problems introduced by others will not reflect on the original
|
||||
authors' reputations.
|
||||
|
||||
Finally, any free program is threatened constantly by software
|
||||
patents. We wish to avoid the danger that redistributors of a free
|
||||
program will individually obtain patent licenses, in effect making the
|
||||
program proprietary. To prevent this, we have made it clear that any
|
||||
patent must be licensed for everyone's free use or not licensed at all.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
|
||||
|
||||
0. This License applies to any program or other work which contains
|
||||
a notice placed by the copyright holder saying it may be distributed
|
||||
under the terms of this General Public License. The "Program", below,
|
||||
refers to any such program or work, and a "work based on the Program"
|
||||
means either the Program or any derivative work under copyright law:
|
||||
that is to say, a work containing the Program or a portion of it,
|
||||
either verbatim or with modifications and/or translated into another
|
||||
language. (Hereinafter, translation is included without limitation in
|
||||
the term "modification".) Each licensee is addressed as "you".
|
||||
|
||||
Activities other than copying, distribution and modification are not
|
||||
covered by this License; they are outside its scope. The act of
|
||||
running the Program is not restricted, and the output from the Program
|
||||
is covered only if its contents constitute a work based on the
|
||||
Program (independent of having been made by running the Program).
|
||||
Whether that is true depends on what the Program does.
|
||||
|
||||
1. You may copy and distribute verbatim copies of the Program's
|
||||
source code as you receive it, in any medium, provided that you
|
||||
conspicuously and appropriately publish on each copy an appropriate
|
||||
copyright notice and disclaimer of warranty; keep intact all the
|
||||
notices that refer to this License and to the absence of any warranty;
|
||||
and give any other recipients of the Program a copy of this License
|
||||
along with the Program.
|
||||
|
||||
You may charge a fee for the physical act of transferring a copy, and
|
||||
you may at your option offer warranty protection in exchange for a fee.
|
||||
|
||||
2. You may modify your copy or copies of the Program or any portion
|
||||
of it, thus forming a work based on the Program, and copy and
|
||||
distribute such modifications or work under the terms of Section 1
|
||||
above, provided that you also meet all of these conditions:
|
||||
|
||||
a) You must cause the modified files to carry prominent notices
|
||||
stating that you changed the files and the date of any change.
|
||||
|
||||
b) You must cause any work that you distribute or publish, that in
|
||||
whole or in part contains or is derived from the Program or any
|
||||
part thereof, to be licensed as a whole at no charge to all third
|
||||
parties under the terms of this License.
|
||||
|
||||
c) If the modified program normally reads commands interactively
|
||||
when run, you must cause it, when started running for such
|
||||
interactive use in the most ordinary way, to print or display an
|
||||
announcement including an appropriate copyright notice and a
|
||||
notice that there is no warranty (or else, saying that you provide
|
||||
a warranty) and that users may redistribute the program under
|
||||
these conditions, and telling the user how to view a copy of this
|
||||
License. (Exception: if the Program itself is interactive but
|
||||
does not normally print such an announcement, your work based on
|
||||
the Program is not required to print an announcement.)
|
||||
|
||||
These requirements apply to the modified work as a whole. If
|
||||
identifiable sections of that work are not derived from the Program,
|
||||
and can be reasonably considered independent and separate works in
|
||||
themselves, then this License, and its terms, do not apply to those
|
||||
sections when you distribute them as separate works. But when you
|
||||
distribute the same sections as part of a whole which is a work based
|
||||
on the Program, the distribution of the whole must be on the terms of
|
||||
this License, whose permissions for other licensees extend to the
|
||||
entire whole, and thus to each and every part regardless of who wrote it.
|
||||
|
||||
Thus, it is not the intent of this section to claim rights or contest
|
||||
your rights to work written entirely by you; rather, the intent is to
|
||||
exercise the right to control the distribution of derivative or
|
||||
collective works based on the Program.
|
||||
|
||||
In addition, mere aggregation of another work not based on the Program
|
||||
with the Program (or with a work based on the Program) on a volume of
|
||||
a storage or distribution medium does not bring the other work under
|
||||
the scope of this License.
|
||||
|
||||
3. You may copy and distribute the Program (or a work based on it,
|
||||
under Section 2) in object code or executable form under the terms of
|
||||
Sections 1 and 2 above provided that you also do one of the following:
|
||||
|
||||
a) Accompany it with the complete corresponding machine-readable
|
||||
source code, which must be distributed under the terms of Sections
|
||||
1 and 2 above on a medium customarily used for software interchange; or,
|
||||
|
||||
b) Accompany it with a written offer, valid for at least three
|
||||
years, to give any third party, for a charge no more than your
|
||||
cost of physically performing source distribution, a complete
|
||||
machine-readable copy of the corresponding source code, to be
|
||||
distributed under the terms of Sections 1 and 2 above on a medium
|
||||
customarily used for software interchange; or,
|
||||
|
||||
c) Accompany it with the information you received as to the offer
|
||||
to distribute corresponding source code. (This alternative is
|
||||
allowed only for noncommercial distribution and only if you
|
||||
received the program in object code or executable form with such
|
||||
an offer, in accord with Subsection b above.)
|
||||
|
||||
The source code for a work means the preferred form of the work for
|
||||
making modifications to it. For an executable work, complete source
|
||||
code means all the source code for all modules it contains, plus any
|
||||
associated interface definition files, plus the scripts used to
|
||||
control compilation and installation of the executable. However, as a
|
||||
special exception, the source code distributed need not include
|
||||
anything that is normally distributed (in either source or binary
|
||||
form) with the major components (compiler, kernel, and so on) of the
|
||||
operating system on which the executable runs, unless that component
|
||||
itself accompanies the executable.
|
||||
|
||||
If distribution of executable or object code is made by offering
|
||||
access to copy from a designated place, then offering equivalent
|
||||
access to copy the source code from the same place counts as
|
||||
distribution of the source code, even though third parties are not
|
||||
compelled to copy the source along with the object code.
|
||||
|
||||
4. You may not copy, modify, sublicense, or distribute the Program
|
||||
except as expressly provided under this License. Any attempt
|
||||
otherwise to copy, modify, sublicense or distribute the Program is
|
||||
void, and will automatically terminate your rights under this License.
|
||||
However, parties who have received copies, or rights, from you under
|
||||
this License will not have their licenses terminated so long as such
|
||||
parties remain in full compliance.
|
||||
|
||||
5. You are not required to accept this License, since you have not
|
||||
signed it. However, nothing else grants you permission to modify or
|
||||
distribute the Program or its derivative works. These actions are
|
||||
prohibited by law if you do not accept this License. Therefore, by
|
||||
modifying or distributing the Program (or any work based on the
|
||||
Program), you indicate your acceptance of this License to do so, and
|
||||
all its terms and conditions for copying, distributing or modifying
|
||||
the Program or works based on it.
|
||||
|
||||
6. Each time you redistribute the Program (or any work based on the
|
||||
Program), the recipient automatically receives a license from the
|
||||
original licensor to copy, distribute or modify the Program subject to
|
||||
these terms and conditions. You may not impose any further
|
||||
restrictions on the recipients' exercise of the rights granted herein.
|
||||
You are not responsible for enforcing compliance by third parties to
|
||||
this License.
|
||||
|
||||
7. If, as a consequence of a court judgment or allegation of patent
|
||||
infringement or for any other reason (not limited to patent issues),
|
||||
conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot
|
||||
distribute so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you
|
||||
may not distribute the Program at all. For example, if a patent
|
||||
license would not permit royalty-free redistribution of the Program by
|
||||
all those who receive copies directly or indirectly through you, then
|
||||
the only way you could satisfy both it and this License would be to
|
||||
refrain entirely from distribution of the Program.
|
||||
|
||||
If any portion of this section is held invalid or unenforceable under
|
||||
any particular circumstance, the balance of the section is intended to
|
||||
apply and the section as a whole is intended to apply in other
|
||||
circumstances.
|
||||
|
||||
It is not the purpose of this section to induce you to infringe any
|
||||
patents or other property right claims or to contest validity of any
|
||||
such claims; this section has the sole purpose of protecting the
|
||||
integrity of the free software distribution system, which is
|
||||
implemented by public license practices. Many people have made
|
||||
generous contributions to the wide range of software distributed
|
||||
through that system in reliance on consistent application of that
|
||||
system; it is up to the author/donor to decide if he or she is willing
|
||||
to distribute software through any other system and a licensee cannot
|
||||
impose that choice.
|
||||
|
||||
This section is intended to make thoroughly clear what is believed to
|
||||
be a consequence of the rest of this License.
|
||||
|
||||
8. If the distribution and/or use of the Program is restricted in
|
||||
certain countries either by patents or by copyrighted interfaces, the
|
||||
original copyright holder who places the Program under this License
|
||||
may add an explicit geographical distribution limitation excluding
|
||||
those countries, so that distribution is permitted only in or among
|
||||
countries not thus excluded. In such case, this License incorporates
|
||||
the limitation as if written in the body of this License.
|
||||
|
||||
9. The Free Software Foundation may publish revised and/or new versions
|
||||
of the General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the Program
|
||||
specifies a version number of this License which applies to it and "any
|
||||
later version", you have the option of following the terms and conditions
|
||||
either of that version or of any later version published by the Free
|
||||
Software Foundation. If the Program does not specify a version number of
|
||||
this License, you may choose any version ever published by the Free Software
|
||||
Foundation.
|
||||
|
||||
10. If you wish to incorporate parts of the Program into other free
|
||||
programs whose distribution conditions are different, write to the author
|
||||
to ask for permission. For software which is copyrighted by the Free
|
||||
Software Foundation, write to the Free Software Foundation; we sometimes
|
||||
make exceptions for this. Our decision will be guided by the two goals
|
||||
of preserving the free status of all derivatives of our free software and
|
||||
of promoting the sharing and reuse of software generally.
|
||||
|
||||
NO WARRANTY
|
||||
|
||||
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
|
||||
FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
|
||||
OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
|
||||
PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
|
||||
OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
|
||||
TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
|
||||
PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
|
||||
REPAIR OR CORRECTION.
|
||||
|
||||
12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
|
||||
REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
|
||||
INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
|
||||
OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
|
||||
TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
|
||||
YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
|
||||
PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGES.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
convey the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along
|
||||
with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program is interactive, make it output a short notice like this
|
||||
when it starts in an interactive mode:
|
||||
|
||||
Gnomovision version 69, Copyright (C) year name of author
|
||||
Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, the commands you use may
|
||||
be called something other than `show w' and `show c'; they could even be
|
||||
mouse-clicks or menu items--whatever suits your program.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or your
|
||||
school, if any, to sign a "copyright disclaimer" for the program, if
|
||||
necessary. Here is a sample; alter the names:
|
||||
|
||||
Yoyodyne, Inc., hereby disclaims all copyright interest in the program
|
||||
`Gnomovision' (which makes passes at compilers) written by James Hacker.
|
||||
|
||||
<signature of Ty Coon>, 1 April 1989
|
||||
Ty Coon, President of Vice
|
||||
|
||||
This General Public License does not permit incorporating your program into
|
||||
proprietary programs. If your program is a subroutine library, you may
|
||||
consider it more useful to permit linking proprietary applications with the
|
||||
library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License.
|
75
third_party/xxhash/tests/collisions/Makefile
vendored
Normal file
75
third_party/xxhash/tests/collisions/Makefile
vendored
Normal file
|
@ -0,0 +1,75 @@
|
|||
# Brute force collision tester for 64-bit hashes
|
||||
# Part of xxHash project
|
||||
# Copyright (C) 2019-2021 Yann Collet
|
||||
#
|
||||
# GPL v2 License
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along
|
||||
# with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
#
|
||||
# You can contact the author at:
|
||||
# - xxHash homepage: https://www.xxhash.com
|
||||
# - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
#
|
||||
|
||||
SRC_DIRS = ./ ../../ allcodecs/
|
||||
VPATH = $(SRC_DIRS)
|
||||
CPPFLAGS += $(addprefix -I ,$(SRC_DIRS))
|
||||
CFLAGS += -Wall -Wextra -Wconversion \
|
||||
-std=c99
|
||||
CXXFLAGS += -Wall -Wextra -Wconversion \
|
||||
-std=c++11
|
||||
LDFLAGS += -pthread
|
||||
TESTHASHES = 110000000
|
||||
|
||||
HASH_SRC := $(sort $(wildcard allcodecs/*.c allcodecs/*.cc))
|
||||
HASH_OBJ := $(patsubst %.c,%.o,$(HASH_SRC))
|
||||
|
||||
|
||||
.PHONY: default
|
||||
default: release
|
||||
|
||||
.PHONY: all
|
||||
all: release
|
||||
|
||||
collisionsTest: main.o pool.o threading.o sort.o $(HASH_OBJ)
|
||||
$(CXX) $(CPPFLAGS) $(CXXFLAGS) $^ $(LDFLAGS) -o $@
|
||||
|
||||
main.o: hashes.h xxhash.h
|
||||
|
||||
release: CXXFLAGS += -O3
|
||||
release: CFLAGS += -O3
|
||||
release: collisionsTest
|
||||
|
||||
debug: CXXFLAGS += -g3 -O0 -DDEBUG
|
||||
debug: CFLAGS += -g3 -O0 -DDEBUG
|
||||
debug: collisionsTest
|
||||
|
||||
.PHONY: check
|
||||
check: test
|
||||
|
||||
.PHONY: test
|
||||
test: debug
|
||||
@echo ""
|
||||
@echo "## $(TESTHASHES) hashes with original and 0 threads"
|
||||
@time ./collisionsTest --nbh=$(TESTHASHES)
|
||||
@echo ""
|
||||
@echo "## $(TESTHASHES) hashes with original and 4 threads"
|
||||
@time ./collisionsTest --nbh=$(TESTHASHES) --threadlog=2
|
||||
@echo ""
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
$(RM) *.o allcodecs/*.o
|
||||
$(RM) collisionsTest
|
122
third_party/xxhash/tests/collisions/README.md
vendored
Normal file
122
third_party/xxhash/tests/collisions/README.md
vendored
Normal file
|
@ -0,0 +1,122 @@
|
|||
|
||||
__collisionsTest__ is a brute force hash analyzer
|
||||
which will measure a 64-bit hash algorithm's collision rate
|
||||
by generating billions of hashes,
|
||||
and comparing the result to an "ideal" target.
|
||||
|
||||
The test requires a very large amount of memory.
|
||||
By default, it will generate 24 billion of 64-bit hashes,
|
||||
requiring __192 GB of RAM__ for their storage.
|
||||
The number of hashes can be modified using command `--nbh=`.
|
||||
Be aware that testing the collision ratio of 64-bit hashes
|
||||
requires a very large amount of hashes (several billion) for meaningful measurements.
|
||||
|
||||
To reduce RAM usage, an optional filter can be requested, with `--filter`.
|
||||
It reduces the nb of candidates to analyze, hence associated RAM budget.
|
||||
Note that the filter itself requires a lot of RAM
|
||||
(32 GB by default, can be modified using `--filterlog=`,
|
||||
a too small filter will not be efficient, aim at ~2 bytes per hash),
|
||||
and reading and writing into filter cost a significant CPU budget,
|
||||
so this method is slower.
|
||||
It also doesn't allow advanced analysis of partial bitfields,
|
||||
since most hashes will be discarded and not stored.
|
||||
|
||||
When using the filter, the RAM budget consists of the filter and a list of candidates,
|
||||
which will be a fraction of the original hash list.
|
||||
Using default settings (24 billion hashes, 32 GB filter),
|
||||
the number of potential candidates should be reduced to less than 2 billion,
|
||||
requiring ~14 GB for their storage.
|
||||
Such a result also depends on hash algorithm's efficiency.
|
||||
The number of effective candidates is likely to be lower, at ~ 1 billion,
|
||||
but storage must allocate an upper bound.
|
||||
|
||||
For the default test, the expected "optimal" collision rate for a 64-bit hash function is ~18 collisions.
|
||||
|
||||
#### How to build
|
||||
```
|
||||
make
|
||||
```
|
||||
|
||||
Note: the code is a mix of C99 and C++14,
|
||||
it's not compatible with a C90-only compiler.
|
||||
|
||||
#### Build modifier
|
||||
|
||||
- `SLAB5`: use alternative pattern generator, friendlier for weak hash algorithms
|
||||
- `POOL_MT`: if `=0`, disable multi-threading code (enabled by default)
|
||||
|
||||
#### How to integrate any hash in the tester
|
||||
|
||||
The build script will compile files found in `./allcodecs`.
|
||||
Put the source code here.
|
||||
This also works if the hash is a single `*.h` file.
|
||||
|
||||
The glue happens in `hashes.h`.
|
||||
In this file, there are 2 sections:
|
||||
- Adds the required `#include "header.h"`, and creates a wrapper
|
||||
to respect the format expected by the function pointer.
|
||||
- Adds the wrapper, along with the name and an indication of the output width,
|
||||
to the table, at the end of `hashes.h`
|
||||
|
||||
Build with `make`. Locate your new hash with `./collisionsTest -h`,
|
||||
it should be listed.
|
||||
|
||||
|
||||
#### Usage
|
||||
|
||||
```
|
||||
usage: ./collisionsTest [hashName] [opt]
|
||||
|
||||
list of hashNames: (...)
|
||||
|
||||
Optional parameters:
|
||||
--nbh=NB Select nb of hashes to generate (25769803776 by default)
|
||||
--filter Enable the filter. Slower, but reduces memory usage for same nb of hashes.
|
||||
--threadlog=NB Use 2^NB threads
|
||||
--len=NB Select length of input (255 bytes by default)
|
||||
```
|
||||
|
||||
#### Some advises on how to setup a collisions test
|
||||
|
||||
Most tests are primarily driven by the amount of RAM available.
|
||||
Here's a method to decide the size of the test.
|
||||
|
||||
Presuming that RAM budget is not plentiful, for this example 32 GB,
|
||||
the `--filter` mode is actually compulsory to measure anything meaningful.
|
||||
Let's plan 50% of memory for the filter, that's 16 GB.
|
||||
This will be good enough to filter about 10% less hashes than this size.
|
||||
Let's round down to 14 G.
|
||||
|
||||
By requesting 14G, the expectation is that the program will automatically
|
||||
size the filter to 16 GB, and expect to store ~1G candidates,
|
||||
leaving enough room to breeze for the system.
|
||||
|
||||
The command line becomes:
|
||||
```
|
||||
./collisionsTest --nbh=14G --filter NameOfHash
|
||||
```
|
||||
|
||||
#### Examples:
|
||||
|
||||
Here are a few results produced with this tester:
|
||||
|
||||
| Algorithm | Input Len | Nb Hashes | Expected | Nb Collisions | Notes |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| __XXH3__ | 255 | 100 Gi | 312.5 | 326 | |
|
||||
| __XXH64__ | 255 | 100 Gi | 312.5 | 294 | |
|
||||
| __XXH128__ low64 | 512 | 100 Gi | 312.5 | 321 | |
|
||||
| __XXH128__ high64| 512 | 100 Gi | 312.5 | 325 | |
|
||||
| __XXH128__ | 255 | 100 Gi | 0.0 | 0 | a 128-bit hash is expected to generate 0 collisions |
|
||||
|
||||
Test on small inputs:
|
||||
|
||||
| Algorithm | Input Len | Nb Hashes | Expected | Nb Collisions | Notes |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| __XXH64__ | 8 | 100 Gi | 312.5 | __0__ | `XXH64` is bijective for `len==8` |
|
||||
| __XXH3__ | 8 | 100 Gi | 312.5 | __0__ | `XXH3` is also bijective for `len==8` |
|
||||
| __XXH3__ | 16 | 100 Gi | 312.5 | 332 | |
|
||||
| __XXH3__ | 32 | 14 Gi | 6.1 | 3 | |
|
||||
| __XXH128__ | 16 | 25 Gi | 0.0 | 0 | test range 9-16 |
|
||||
| __XXH128__ | 32 | 25 Gi | 0.0 | 0 | test range 17-128 |
|
||||
| __XXH128__ | 100 | 13 Gi | 0.0 | 0 | test range 17-128 |
|
||||
| __XXH128__ | 200 | 13 Gi | 0.0 | 0 | test range 129-240 |
|
1
third_party/xxhash/tests/collisions/allcodecs/README.md
vendored
Normal file
1
third_party/xxhash/tests/collisions/allcodecs/README.md
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
Put in this directory all hash algorithms to test
|
38
third_party/xxhash/tests/collisions/allcodecs/dummy.c
vendored
Normal file
38
third_party/xxhash/tests/collisions/allcodecs/dummy.c
vendored
Normal file
|
@ -0,0 +1,38 @@
|
|||
/*
|
||||
* dummy.c, a fake hash algorithm, just to test integration capabilities.
|
||||
* Part of the xxHash project
|
||||
* Copyright (C) 2020 Yann Collet
|
||||
*
|
||||
* GPL v2 License
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along
|
||||
* with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - xxHash homepage: https://www.xxhash.com
|
||||
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
|
||||
#include <dummy.h>
|
||||
|
||||
unsigned badsum32(const void* input, size_t len, unsigned seed)
|
||||
{
|
||||
unsigned sum = seed;
|
||||
const unsigned char* in8 = input;
|
||||
size_t c;
|
||||
for (c=0; c<len; c++)
|
||||
sum += in8[c];
|
||||
return sum;
|
||||
}
|
45
third_party/xxhash/tests/collisions/allcodecs/dummy.h
vendored
Normal file
45
third_party/xxhash/tests/collisions/allcodecs/dummy.h
vendored
Normal file
|
@ -0,0 +1,45 @@
|
|||
/*
|
||||
* dummy.c,
|
||||
* A fake hash algorithm, just to test integration capabilities.
|
||||
* Part of the xxHash project
|
||||
* Copyright (C) 2020 Yann Collet
|
||||
*
|
||||
* GPL v2 License
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along
|
||||
* with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - xxHash homepage: https://www.xxhash.com
|
||||
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
#ifndef DUMMY_H_987987
|
||||
#define DUMMY_H_987987
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
unsigned badsum32(const void* input, size_t len, unsigned seed);
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* DUMMY_H_987987 */
|
127
third_party/xxhash/tests/collisions/hashes.h
vendored
Normal file
127
third_party/xxhash/tests/collisions/hashes.h
vendored
Normal file
|
@ -0,0 +1,127 @@
|
|||
/*
|
||||
* List of hashes for the brute force collision tester
|
||||
* Part of xxHash project
|
||||
* Copyright (C) 2019-2021 Yann Collet
|
||||
*
|
||||
* GPL v2 License
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along
|
||||
* with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - xxHash homepage: https://www.xxhash.com
|
||||
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
#ifndef HASHES_H_1235465
|
||||
#define HASHES_H_1235465
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdint.h> /* uint64_t */
|
||||
#define XXH_INLINE_ALL /* XXH128_hash_t */
|
||||
#include "xxhash.h"
|
||||
|
||||
|
||||
/* return type */
|
||||
|
||||
typedef union {
|
||||
uint64_t h64;
|
||||
XXH128_hash_t h128;
|
||||
} UniHash;
|
||||
|
||||
UniHash uniHash32(uint64_t v32)
|
||||
{ UniHash unih;
|
||||
unih.h64 = v32;
|
||||
return unih;
|
||||
}
|
||||
|
||||
UniHash uniHash64(uint64_t v64)
|
||||
{ UniHash unih;
|
||||
unih.h64 = v64;
|
||||
return unih;
|
||||
}
|
||||
|
||||
UniHash uniHash128(XXH128_hash_t v128)
|
||||
{ UniHash unih;
|
||||
unih.h128 = v128;
|
||||
return unih;
|
||||
}
|
||||
|
||||
|
||||
/* === xxHash === */
|
||||
|
||||
UniHash XXH3_wrapper (const void* data, size_t size)
|
||||
{
|
||||
return uniHash64( XXH3_64bits(data, size) );
|
||||
}
|
||||
|
||||
UniHash XXH128_wrapper (const void* data, size_t size)
|
||||
{
|
||||
return uniHash128( XXH3_128bits(data, size) );
|
||||
}
|
||||
|
||||
UniHash XXH128l_wrapper (const void* data, size_t size)
|
||||
{
|
||||
return uniHash64( XXH3_128bits(data, size).low64 );
|
||||
}
|
||||
|
||||
UniHash XXH128h_wrapper (const void* data, size_t size)
|
||||
{
|
||||
return uniHash64( XXH3_128bits(data, size).high64 );
|
||||
}
|
||||
|
||||
UniHash XXH64_wrapper (const void* data, size_t size)
|
||||
{
|
||||
return uniHash64 ( XXH64(data, size, 0) );
|
||||
}
|
||||
|
||||
UniHash XXH32_wrapper (const void* data, size_t size)
|
||||
{
|
||||
return uniHash32( XXH32(data, size, 0) );
|
||||
}
|
||||
|
||||
/* === Dummy integration example === */
|
||||
|
||||
#include "dummy.h"
|
||||
|
||||
UniHash badsum32_wrapper (const void* data, size_t size)
|
||||
{
|
||||
return uniHash32( badsum32(data, size, 0) );
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* === Table === */
|
||||
|
||||
typedef UniHash (*hashfn) (const void* data, size_t size);
|
||||
|
||||
typedef struct {
|
||||
const char* name;
|
||||
hashfn fn;
|
||||
int bits;
|
||||
} hashDescription;
|
||||
|
||||
#define HASH_FN_TOTAL 7
|
||||
|
||||
hashDescription hashfnTable[HASH_FN_TOTAL] = {
|
||||
{ "xxh3" , XXH3_wrapper, 64 },
|
||||
{ "xxh64" , XXH64_wrapper, 64 },
|
||||
{ "xxh128", XXH128_wrapper, 128 },
|
||||
{ "xxh128l", XXH128l_wrapper, 64 },
|
||||
{ "xxh128h", XXH128h_wrapper, 64 },
|
||||
{ "xxh32" , XXH32_wrapper, 32 },
|
||||
{ "badsum32",badsum32_wrapper, 32 },
|
||||
};
|
||||
|
||||
#endif /* HASHES_H_1235465 */
|
1124
third_party/xxhash/tests/collisions/main.c
vendored
Normal file
1124
third_party/xxhash/tests/collisions/main.c
vendored
Normal file
File diff suppressed because it is too large
Load diff
344
third_party/xxhash/tests/collisions/pool.c
vendored
Normal file
344
third_party/xxhash/tests/collisions/pool.c
vendored
Normal file
|
@ -0,0 +1,344 @@
|
|||
/*
|
||||
* Copyright (C) 2016-2021 Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
|
||||
/* ====== Dependencies ======= */
|
||||
#include <stddef.h> /* size_t */
|
||||
#include <stdlib.h> /* malloc, calloc, free */
|
||||
#include <string.h> /* memcpy */
|
||||
#include <assert.h>
|
||||
|
||||
#include "pool.h"
|
||||
|
||||
|
||||
/* ====== Compiler specifics ====== */
|
||||
#if defined(_MSC_VER)
|
||||
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
|
||||
#endif
|
||||
|
||||
|
||||
/* === Build Macro === */
|
||||
|
||||
#ifndef POOL_MT // can be defined on command line
|
||||
# define POOL_MT 1
|
||||
#endif
|
||||
|
||||
|
||||
/* === Implementation === */
|
||||
|
||||
#if POOL_MT
|
||||
|
||||
#include "threading.h" /* pthread adaptation */
|
||||
|
||||
/* A job is a function and an opaque argument */
|
||||
typedef struct POOL_job_s {
|
||||
POOL_function function;
|
||||
void *opaque;
|
||||
} POOL_job;
|
||||
|
||||
struct POOL_ctx_s {
|
||||
/* Keep track of the threads */
|
||||
ZSTD_pthread_t* threads;
|
||||
size_t threadCapacity;
|
||||
size_t threadLimit;
|
||||
|
||||
/* The queue is a circular buffer */
|
||||
POOL_job *queue;
|
||||
size_t queueHead;
|
||||
size_t queueTail;
|
||||
size_t queueSize;
|
||||
|
||||
/* The number of threads working on jobs */
|
||||
size_t numThreadsBusy;
|
||||
/* Indicates if the queue is empty */
|
||||
int queueEmpty;
|
||||
|
||||
/* The mutex protects the queue */
|
||||
ZSTD_pthread_mutex_t queueMutex;
|
||||
/* Condition variable for pushers to wait on when the queue is full */
|
||||
ZSTD_pthread_cond_t queuePushCond;
|
||||
/* Condition variables for poppers to wait on when the queue is empty */
|
||||
ZSTD_pthread_cond_t queuePopCond;
|
||||
/* Indicates if the queue is shutting down */
|
||||
int shutdown;
|
||||
};
|
||||
|
||||
/* POOL_thread() :
|
||||
* Work thread for the thread pool.
|
||||
* Waits for jobs and executes them.
|
||||
* @returns : NULL on failure else non-null.
|
||||
*/
|
||||
static void* POOL_thread(void* opaque)
|
||||
{
|
||||
POOL_ctx* const ctx = (POOL_ctx*)opaque;
|
||||
if (!ctx) { return NULL; }
|
||||
for (;;) {
|
||||
/* Lock the mutex and wait for a non-empty queue or until shutdown */
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
|
||||
while ( ctx->queueEmpty
|
||||
|| (ctx->numThreadsBusy >= ctx->threadLimit) ) {
|
||||
if (ctx->shutdown) {
|
||||
/* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit),
|
||||
* a few threads will be shutdown while !queueEmpty,
|
||||
* but enough threads will remain active to finish the queue */
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
return opaque;
|
||||
}
|
||||
ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
|
||||
}
|
||||
/* Pop a job off the queue */
|
||||
{ POOL_job const job = ctx->queue[ctx->queueHead];
|
||||
ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize;
|
||||
ctx->numThreadsBusy++;
|
||||
ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
|
||||
/* Unlock the mutex, signal a pusher, and run the job */
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
|
||||
job.function(job.opaque);
|
||||
|
||||
/* If the intended queue size was 0, signal after finishing job */
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
ctx->numThreadsBusy--;
|
||||
if (ctx->queueSize == 1) {
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
|
||||
}
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
}
|
||||
} /* for (;;) */
|
||||
assert(0); /* Unreachable */
|
||||
}
|
||||
|
||||
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize)
|
||||
{
|
||||
POOL_ctx* ctx;
|
||||
/* Check parameters */
|
||||
if (!numThreads) { return NULL; }
|
||||
/* Allocate the context and zero initialize */
|
||||
ctx = (POOL_ctx*)calloc(1, sizeof(POOL_ctx));
|
||||
if (!ctx) { return NULL; }
|
||||
/* Initialize the job queue.
|
||||
* It needs one extra space since one space is wasted to differentiate
|
||||
* empty and full queues.
|
||||
*/
|
||||
ctx->queueSize = queueSize + 1;
|
||||
ctx->queue = (POOL_job*)malloc(ctx->queueSize * sizeof(POOL_job));
|
||||
ctx->queueHead = 0;
|
||||
ctx->queueTail = 0;
|
||||
ctx->numThreadsBusy = 0;
|
||||
ctx->queueEmpty = 1;
|
||||
(void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
|
||||
(void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
|
||||
ctx->shutdown = 0;
|
||||
/* Allocate space for the thread handles */
|
||||
ctx->threads = (ZSTD_pthread_t*)malloc(numThreads * sizeof(ZSTD_pthread_t));
|
||||
ctx->threadCapacity = 0;
|
||||
/* Check for errors */
|
||||
if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; }
|
||||
/* Initialize the threads */
|
||||
{ size_t i;
|
||||
for (i = 0; i < numThreads; ++i) {
|
||||
if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) {
|
||||
ctx->threadCapacity = i;
|
||||
POOL_free(ctx);
|
||||
return NULL;
|
||||
} }
|
||||
ctx->threadCapacity = numThreads;
|
||||
ctx->threadLimit = numThreads;
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
/*! POOL_join() :
|
||||
Shutdown the queue, wake any sleeping threads, and join all of the threads.
|
||||
*/
|
||||
static void POOL_join(POOL_ctx* ctx) {
|
||||
/* Shut down the queue */
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
ctx->shutdown = 1;
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
|
||||
/* Wake up sleeping threads */
|
||||
ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
|
||||
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
||||
|
||||
/* Join all of the threads */
|
||||
{ size_t i;
|
||||
for (i = 0; i < ctx->threadCapacity; ++i) {
|
||||
ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */
|
||||
} }
|
||||
}
|
||||
|
||||
void POOL_free(POOL_ctx *ctx) {
|
||||
if (!ctx) { return; }
|
||||
POOL_join(ctx);
|
||||
ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
|
||||
ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
|
||||
ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
|
||||
free(ctx->queue);
|
||||
free(ctx->threads);
|
||||
free(ctx);
|
||||
}
|
||||
|
||||
|
||||
|
||||
size_t POOL_sizeof(POOL_ctx *ctx) {
|
||||
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
||||
return sizeof(*ctx)
|
||||
+ ctx->queueSize * sizeof(POOL_job)
|
||||
+ ctx->threadCapacity * sizeof(ZSTD_pthread_t);
|
||||
}
|
||||
|
||||
|
||||
/* @return : 0 on success, 1 on error */
|
||||
static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads)
|
||||
{
|
||||
if (numThreads <= ctx->threadCapacity) {
|
||||
if (!numThreads) return 1;
|
||||
ctx->threadLimit = numThreads;
|
||||
return 0;
|
||||
}
|
||||
/* numThreads > threadCapacity */
|
||||
{ ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)malloc(numThreads * sizeof(ZSTD_pthread_t));
|
||||
if (!threadPool) return 1;
|
||||
/* replace existing thread pool */
|
||||
memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool));
|
||||
free(ctx->threads);
|
||||
ctx->threads = threadPool;
|
||||
/* Initialize additional threads */
|
||||
{ size_t threadId;
|
||||
for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) {
|
||||
if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) {
|
||||
ctx->threadCapacity = threadId;
|
||||
return 1;
|
||||
} }
|
||||
} }
|
||||
/* successfully expanded */
|
||||
ctx->threadCapacity = numThreads;
|
||||
ctx->threadLimit = numThreads;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* @return : 0 on success, 1 on error */
|
||||
int POOL_resize(POOL_ctx* ctx, size_t numThreads)
|
||||
{
|
||||
int result;
|
||||
if (ctx==NULL) return 1;
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
result = POOL_resize_internal(ctx, numThreads);
|
||||
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns 1 if the queue is full and 0 otherwise.
|
||||
*
|
||||
* When queueSize is 1 (pool was created with an intended queueSize of 0),
|
||||
* then a queue is empty if there is a thread free _and_ no job is waiting.
|
||||
*/
|
||||
static int isQueueFull(POOL_ctx const* ctx) {
|
||||
if (ctx->queueSize > 1) {
|
||||
return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize);
|
||||
} else {
|
||||
return (ctx->numThreadsBusy == ctx->threadLimit) ||
|
||||
!ctx->queueEmpty;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque)
|
||||
{
|
||||
POOL_job const job = {function, opaque};
|
||||
assert(ctx != NULL);
|
||||
if (ctx->shutdown) return;
|
||||
|
||||
ctx->queueEmpty = 0;
|
||||
ctx->queue[ctx->queueTail] = job;
|
||||
ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize;
|
||||
ZSTD_pthread_cond_signal(&ctx->queuePopCond);
|
||||
}
|
||||
|
||||
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque)
|
||||
{
|
||||
assert(ctx != NULL);
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
/* Wait until there is space in the queue for the new job */
|
||||
while (isQueueFull(ctx) && (!ctx->shutdown)) {
|
||||
ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
|
||||
}
|
||||
POOL_add_internal(ctx, function, opaque);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
}
|
||||
|
||||
|
||||
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque)
|
||||
{
|
||||
assert(ctx != NULL);
|
||||
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
|
||||
if (isQueueFull(ctx)) {
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
return 0;
|
||||
}
|
||||
POOL_add_internal(ctx, function, opaque);
|
||||
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
#else /* POOL_MT not defined */
|
||||
|
||||
/* ========================== */
|
||||
/* No multi-threading support */
|
||||
/* ========================== */
|
||||
|
||||
|
||||
/* We don't need any data, but if it is empty, malloc() might return NULL. */
|
||||
struct POOL_ctx_s {
|
||||
int dummy;
|
||||
};
|
||||
static POOL_ctx g_ctx;
|
||||
|
||||
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) {
|
||||
(void)numThreads;
|
||||
(void)queueSize;
|
||||
return &g_ctx;
|
||||
}
|
||||
|
||||
void POOL_free(POOL_ctx* ctx) {
|
||||
assert(!ctx || ctx == &g_ctx);
|
||||
(void)ctx;
|
||||
}
|
||||
|
||||
int POOL_resize(POOL_ctx* ctx, size_t numThreads) {
|
||||
(void)ctx; (void)numThreads;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
||||
(void)ctx;
|
||||
function(opaque);
|
||||
}
|
||||
|
||||
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) {
|
||||
(void)ctx;
|
||||
function(opaque);
|
||||
return 1;
|
||||
}
|
||||
|
||||
size_t POOL_sizeof(POOL_ctx* ctx) {
|
||||
if (ctx==NULL) return 0; /* supports sizeof NULL */
|
||||
assert(ctx == &g_ctx);
|
||||
return sizeof(*ctx);
|
||||
}
|
||||
|
||||
#endif /* ZSTD_MULTITHREAD */
|
80
third_party/xxhash/tests/collisions/pool.h
vendored
Normal file
80
third_party/xxhash/tests/collisions/pool.h
vendored
Normal file
|
@ -0,0 +1,80 @@
|
|||
/*
|
||||
* Copyright (c) 2016-2021 Yann Collet, Facebook, Inc.
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
* You may select, at your option, one of the above-listed licenses.
|
||||
*/
|
||||
|
||||
#ifndef POOL_H
|
||||
#define POOL_H
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
#include <stddef.h> /* size_t */
|
||||
|
||||
typedef struct POOL_ctx_s POOL_ctx;
|
||||
|
||||
/*! POOL_create() :
|
||||
* Create a thread pool with at most `numThreads` threads.
|
||||
* `numThreads` must be at least 1.
|
||||
* The maximum number of queued jobs before blocking is `queueSize`.
|
||||
* @return : POOL_ctx pointer on success, else NULL.
|
||||
*/
|
||||
POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
|
||||
|
||||
/*! POOL_free() :
|
||||
* Free a thread pool returned by POOL_create().
|
||||
*/
|
||||
void POOL_free(POOL_ctx* ctx);
|
||||
|
||||
/*! POOL_resize() :
|
||||
* Expands or shrinks pool's number of threads.
|
||||
* This is more efficient than releasing + creating a new context,
|
||||
* since it tries to preserve and re-use existing threads.
|
||||
* `numThreads` must be at least 1.
|
||||
* @return : 0 when resize was successful,
|
||||
* !0 (typically 1) if there is an error.
|
||||
* note : only numThreads can be resized, queueSize remains unchanged.
|
||||
*/
|
||||
int POOL_resize(POOL_ctx* ctx, size_t numThreads);
|
||||
|
||||
/*! POOL_sizeof() :
|
||||
* @return threadpool memory usage
|
||||
* note : compatible with NULL (returns 0 in this case)
|
||||
*/
|
||||
size_t POOL_sizeof(POOL_ctx* ctx);
|
||||
|
||||
/*! POOL_function :
|
||||
* The function type that can be added to a thread pool.
|
||||
*/
|
||||
typedef void (*POOL_function)(void*);
|
||||
|
||||
/*! POOL_add() :
|
||||
* Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
|
||||
* Possibly blocks until there is room in the queue.
|
||||
* Note : The function may be executed asynchronously,
|
||||
* therefore, `opaque` must live until function has been completed.
|
||||
*/
|
||||
void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
|
||||
|
||||
|
||||
/*! POOL_tryAdd() :
|
||||
* Add the job `function(opaque)` to thread pool _if_ a worker is available.
|
||||
* Returns immediately even if not (does not block).
|
||||
* @return : 1 if successful, 0 if not.
|
||||
*/
|
||||
int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
|
||||
|
||||
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
59
third_party/xxhash/tests/collisions/sort.cc
vendored
Normal file
59
third_party/xxhash/tests/collisions/sort.cc
vendored
Normal file
|
@ -0,0 +1,59 @@
|
|||
/*
|
||||
* sort.cc - C++ sort functions
|
||||
* Copyright (C) 2019-2021 Yann Collet
|
||||
* GPL v2 License
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along
|
||||
* with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - xxHash homepage: https://www.xxhash.com
|
||||
* - xxHash source repository: https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
/*
|
||||
* C++ sort functions tend to run faster than C ones due to templates allowing
|
||||
* inline optimizations.
|
||||
* Also, glibc's qsort() seems to inflate memory usage, resulting in OOM
|
||||
* crashes on the test server.
|
||||
*/
|
||||
|
||||
#include <algorithm> // std::sort
|
||||
#define XXH_INLINE_ALL // XXH128_cmp
|
||||
#include <xxhash.h>
|
||||
|
||||
#include "sort.hh"
|
||||
|
||||
void sort64(uint64_t* table, size_t size)
|
||||
{
|
||||
std::sort(table, table + size);
|
||||
}
|
||||
|
||||
#include <stdlib.h> // qsort
|
||||
|
||||
void sort128(XXH128_hash_t* table, size_t size)
|
||||
{
|
||||
#if 0
|
||||
// C++ sort using a custom function object
|
||||
struct {
|
||||
bool operator()(XXH128_hash_t a, XXH128_hash_t b) const
|
||||
{
|
||||
return XXH128_cmp(&a, &b);
|
||||
}
|
||||
} customLess;
|
||||
std::sort(table, table + size, customLess);
|
||||
#else
|
||||
qsort(table, size, sizeof(*table), XXH128_cmp);
|
||||
#endif
|
||||
}
|
40
third_party/xxhash/tests/collisions/sort.hh
vendored
Normal file
40
third_party/xxhash/tests/collisions/sort.hh
vendored
Normal file
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* sort.hh - headers for C++ sort functions
|
||||
* Copyright (C) 2019-2021 Yann Collet
|
||||
* GPL v2 License
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation; either version 2 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License along
|
||||
* with this program; if not, write to the Free Software Foundation, Inc.,
|
||||
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
*
|
||||
* You can contact the author at :
|
||||
* - xxHash homepage : https://www.xxhash.com
|
||||
* - xxHash source repository : https://github.com/Cyan4973/xxHash
|
||||
*/
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include <stddef.h> // size
|
||||
#include <stdint.h> // uint64_t
|
||||
#define XXH_STATIC_LINKING_ONLY // XXH128_hash_t
|
||||
#include "xxhash.h"
|
||||
|
||||
void sort64(uint64_t* table, size_t size);
|
||||
|
||||
void sort128(XXH128_hash_t* table, size_t size);
|
||||
|
||||
#ifdef __cplusplus
|
||||
} // extern C
|
||||
#endif
|
82
third_party/xxhash/tests/collisions/threading.c
vendored
Normal file
82
third_party/xxhash/tests/collisions/threading.c
vendored
Normal file
|
@ -0,0 +1,82 @@
|
|||
/**
|
||||
* Copyright (c) 2016 Tino Reichardt
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
|
||||
*/
|
||||
|
||||
/**
|
||||
* This file will hold wrapper for systems, which do not support pthreads
|
||||
*/
|
||||
|
||||
|
||||
/* === Build Macro === */
|
||||
|
||||
#ifndef POOL_MT // can be defined on command line
|
||||
# define POOL_MT 1
|
||||
#endif
|
||||
|
||||
|
||||
/* create fake symbol to avoid empty translation unit warning */
|
||||
int g_ZSTD_threading_useles_symbol;
|
||||
|
||||
#if POOL_MT && defined(_WIN32)
|
||||
|
||||
/**
|
||||
* Windows minimalist Pthread Wrapper
|
||||
*/
|
||||
|
||||
|
||||
/* === Dependencies === */
|
||||
#include <process.h>
|
||||
#include <errno.h>
|
||||
#include "threading.h"
|
||||
|
||||
|
||||
/* === Implementation === */
|
||||
|
||||
static unsigned __stdcall worker(void *arg)
|
||||
{
|
||||
ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
|
||||
thread->arg = thread->start_routine(thread->arg);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
|
||||
void* (*start_routine) (void*), void* arg)
|
||||
{
|
||||
(void)unused;
|
||||
thread->arg = arg;
|
||||
thread->start_routine = start_routine;
|
||||
thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
|
||||
|
||||
if (!thread->handle)
|
||||
return errno;
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
|
||||
{
|
||||
DWORD result;
|
||||
|
||||
if (!thread.handle) return 0;
|
||||
|
||||
result = WaitForSingleObject(thread.handle, INFINITE);
|
||||
switch (result) {
|
||||
case WAIT_OBJECT_0:
|
||||
if (value_ptr) *value_ptr = thread.arg;
|
||||
return 0;
|
||||
case WAIT_ABANDONED:
|
||||
return EINVAL;
|
||||
default:
|
||||
return (int)GetLastError();
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* POOL_MT */
|
124
third_party/xxhash/tests/collisions/threading.h
vendored
Normal file
124
third_party/xxhash/tests/collisions/threading.h
vendored
Normal file
|
@ -0,0 +1,124 @@
|
|||
/**
|
||||
* Copyright (c) 2016 Tino Reichardt
|
||||
* All rights reserved.
|
||||
*
|
||||
* This source code is licensed under both the BSD-style license (found in the
|
||||
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
|
||||
* in the COPYING file in the root directory of this source tree).
|
||||
*
|
||||
* You can contact the author at:
|
||||
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
|
||||
*/
|
||||
|
||||
#ifndef THREADING_H_938743
|
||||
#define THREADING_H_938743
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
/* === Build Macro === */
|
||||
|
||||
#ifndef POOL_MT // can be defined on command line
|
||||
# define POOL_MT 1
|
||||
#endif
|
||||
|
||||
|
||||
/* === Implementation === */
|
||||
|
||||
#if POOL_MT && defined(_WIN32)
|
||||
|
||||
/**
|
||||
* Define windows version before include
|
||||
*/
|
||||
#undef WINVER
|
||||
#define WINVER 0x0600
|
||||
|
||||
#undef _WIN32_WINNT
|
||||
#define _WIN32_WINNT 0x0600
|
||||
|
||||
#ifndef WIN32_LEAN_AND_MEAN
|
||||
# define WIN32_LEAN_AND_MEAN
|
||||
#endif
|
||||
|
||||
#include <windows.h>
|
||||
#include <stdio.h>
|
||||
|
||||
/* mutex */
|
||||
#define ZSTD_pthread_mutex_t CRITICAL_SECTION
|
||||
#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0)
|
||||
#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a))
|
||||
#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a))
|
||||
#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a))
|
||||
|
||||
/* condition variable */
|
||||
#define ZSTD_pthread_cond_t CONDITION_VARIABLE
|
||||
#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0)
|
||||
#define ZSTD_pthread_cond_destroy(a) ((void)(a))
|
||||
#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
|
||||
#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a))
|
||||
#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a))
|
||||
|
||||
/* ZSTD_pthread_create() and ZSTD_pthread_join() */
|
||||
typedef struct {
|
||||
HANDLE handle;
|
||||
void* (*start_routine)(void*);
|
||||
void* arg;
|
||||
} ZSTD_pthread_t;
|
||||
|
||||
int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused,
|
||||
void* (*start_routine) (void*), void* arg);
|
||||
|
||||
int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
|
||||
|
||||
/**
|
||||
* add here more wrappers as required
|
||||
*/
|
||||
|
||||
|
||||
#elif POOL_MT /* posix assumed ; need a better detection method */
|
||||
/* === POSIX Systems === */
|
||||
# include <pthread.h>
|
||||
|
||||
#define ZSTD_pthread_mutex_t pthread_mutex_t
|
||||
#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
|
||||
#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
|
||||
#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a))
|
||||
#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a))
|
||||
|
||||
#define ZSTD_pthread_cond_t pthread_cond_t
|
||||
#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b))
|
||||
#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a))
|
||||
#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b))
|
||||
#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a))
|
||||
#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a))
|
||||
|
||||
#define ZSTD_pthread_t pthread_t
|
||||
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
|
||||
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
|
||||
|
||||
#else /* POOL_MT == 0 */
|
||||
/* No multithreading support */
|
||||
|
||||
typedef int ZSTD_pthread_mutex_t;
|
||||
#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0)
|
||||
#define ZSTD_pthread_mutex_destroy(a) ((void)(a))
|
||||
#define ZSTD_pthread_mutex_lock(a) ((void)(a))
|
||||
#define ZSTD_pthread_mutex_unlock(a) ((void)(a))
|
||||
|
||||
typedef int ZSTD_pthread_cond_t;
|
||||
#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0)
|
||||
#define ZSTD_pthread_cond_destroy(a) ((void)(a))
|
||||
#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b))
|
||||
#define ZSTD_pthread_cond_signal(a) ((void)(a))
|
||||
#define ZSTD_pthread_cond_broadcast(a) ((void)(a))
|
||||
|
||||
/* do not use ZSTD_pthread_t */
|
||||
|
||||
#endif /* POOL_MT */
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* THREADING_H_938743 */
|
Loading…
Add table
Add a link
Reference in a new issue