Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add file locking to support parallel runs. #427

Merged
merged 34 commits into from
Jan 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
da8ba91
Add file locking to support parallel runs.
PercentBoat4164 Nov 27, 2022
6abda4b
Fixed formatting.
PercentBoat4164 Nov 28, 2022
218c079
Prevent double locking file.
PercentBoat4164 Nov 28, 2022
ea6411f
Fix SegFault from test 2.
PercentBoat4164 Nov 28, 2022
665ef68
Remove unnecessary debugging messages.
PercentBoat4164 Nov 28, 2022
f4ad1d6
Lock the package directory rather than the cache directory.
PercentBoat4164 Nov 29, 2022
d657584
Lock the version specific cache entry rather than the package specifi…
PercentBoat4164 Nov 29, 2022
773228e
Remove unnecessary arguments in conditional statements.
PercentBoat4164 Nov 29, 2022
3ecd96a
Change back to locking entire cache directory.
PercentBoat4164 Nov 29, 2022
83a4a51
Only check CPM_HAS_CACHE_LOCK.
PercentBoat4164 Nov 29, 2022
b2fba4e
Lock on a per-package basis rather than the entire cache.
PercentBoat4164 Dec 1, 2022
9d72172
Clean up the locked file.
PercentBoat4164 Dec 1, 2022
2250917
Unlock then remove to fix Windows.
PercentBoat4164 Dec 1, 2022
9f1747f
Merge branch 'cpm-cmake:master' into master
PercentBoat4164 Dec 6, 2022
9f4a952
Specify use of cmake.lock as the lock file.
PercentBoat4164 Dec 6, 2022
9660df6
Merge branch 'master' into master
TheLartians Jan 10, 2023
5ec1db4
Merge branch 'cpm-cmake:master' into master
PercentBoat4164 Jan 10, 2023
579f272
- Changed CPM_HAS_CACHE_LOCK to ${CPM_ARGS_NAME}_CPM_HAS_CACHE_LOCK.
PercentBoat4164 Jan 10, 2023
85f3ebb
Add unit test.
PercentBoat4164 Jan 10, 2023
70e4f57
Actually test if resulting git cache is clean in unit test.
PercentBoat4164 Jan 11, 2023
b2eb097
- Added comments
PercentBoat4164 Jan 11, 2023
937abb1
convert parallelism test to integration test
TheLartians Jan 24, 2023
a5a33b8
remove comment
TheLartians Jan 24, 2023
82c05f5
- Removed now unnecessary variable.
PercentBoat4164 Jan 25, 2023
0397a72
Forgot to change variable name.
PercentBoat4164 Jan 25, 2023
dd2ef70
Add similar changes to the missed section.
PercentBoat4164 Jan 25, 2023
0bc1f2d
Fixed formatting.
PercentBoat4164 Jan 25, 2023
c031a3d
Unlock the file, but do not delete it.
PercentBoat4164 Jan 25, 2023
88592ef
Only unlock the file if it exists.
PercentBoat4164 Jan 25, 2023
42ef7b6
Changed cache.cmake test to ignore non-directory entries.
PercentBoat4164 Jan 25, 2023
bc00dd7
Integration test lib make_project:
iboB Jan 26, 2023
dce53a1
- Moved checks to function.
PercentBoat4164 Jan 26, 2023
5789a87
- Fix formatting
PercentBoat4164 Jan 26, 2023
dffafed
Switch to snake case.
PercentBoat4164 Jan 27, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cmake/CPM.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -712,6 +712,9 @@ function(CPMAddPackage)
# relative paths.
get_filename_component(download_directory ${download_directory} ABSOLUTE)
list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS SOURCE_DIR ${download_directory})

file(LOCK ${download_directory}/../cmake.lock)

if(EXISTS ${download_directory})
cpm_store_fetch_properties(
${CPM_ARGS_NAME} "${download_directory}"
Expand Down Expand Up @@ -790,6 +793,10 @@ function(CPMAddPackage)
cpm_get_fetch_properties("${CPM_ARGS_NAME}")
endif()

if(EXISTS ${download_directory}/../cmake.lock)
file(LOCK ${download_directory}/../cmake.lock RELEASE)
endif()

set(${CPM_ARGS_NAME}_ADDED YES)
cpm_export_variables("${CPM_ARGS_NAME}")
endfunction()
Expand Down
11 changes: 6 additions & 5 deletions test/integration/lib.rb
Original file line number Diff line number Diff line change
Expand Up @@ -180,19 +180,20 @@ def cur_test_dir
@@test_dir
end

def make_project(template_dir = nil)
def make_project(name: nil, from_template: nil)
test_name = local_name
test_name = test_name[5..] if test_name.start_with?('test_')

base = File.join(cur_test_dir, test_name)
base += "-#{name}" if name
src_dir = base + '-src'

FileUtils.mkdir_p src_dir

if template_dir
template_dir = File.join(TestLib::TEMPLATES_DIR, template_dir)
raise "#{template_dir} is not a directory" if !File.directory?(template_dir)
FileUtils.copy_entry template_dir, src_dir
if from_template
from_template = File.join(TestLib::TEMPLATES_DIR, from_template)
raise "#{from_template} is not a directory" if !File.directory?(from_template)
FileUtils.copy_entry from_template, src_dir
end

Project.new src_dir, base + '-bin'
Expand Down
7 changes: 4 additions & 3 deletions test/integration/reference.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Integration Test Framework Refernce
# Integration Test Framework Reference

## `TestLib`

Expand Down Expand Up @@ -60,6 +60,7 @@ The class which must be a parent of all integration test case classes. It itself
### Utils

* `cur_test_dir` - the directory of the current test case. A subdirectory of `TestLib::TMP_DIR`
* `make_project(template_dir = nil)` - create a project from a test method. Will create a the project's source and binary directories as subdirectories of `cur_test_dir`.
* Optionally work with a template directory, in which case it will copy the contents of the template directory (one from `templates`) in the project's source directory.
* `make_project(name: nil, from_template: nil)` - create a project from a test method. Will create the project's source and binary directories as subdirectories of `cur_test_dir`.
* Optionally provide a name which will be concatenated to the project directory. This allows creating multiple projects in a test
* Optionally work with a template, in which case it will copy the contents of the template directory (one from `templates`) in the project's source directory.

4 changes: 2 additions & 2 deletions test/integration/test_basics.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
class Basics < IntegrationTest
# Test cpm caches with no cpm-related env vars
def test_cpm_default
prj = make_project 'no-deps'
prj = make_project from_template: 'no-deps'
prj.create_lists_from_default_template
assert_success prj.configure

Expand Down Expand Up @@ -38,7 +38,7 @@ def test_cpm_default
def test_env_cpm_source_cache
ENV['CPM_SOURCE_CACHE'] = cur_test_dir

prj = make_project 'no-deps'
prj = make_project from_template: 'no-deps'
prj.create_lists_from_default_template
assert_success prj.configure

Expand Down
4 changes: 2 additions & 2 deletions test/integration/test_fetchcontent_compatibility.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ def setup
end

def test_add_dependency_cpm_and_fetchcontent
prj = make_project 'using-adder'
prj = make_project from_template: 'using-adder'

prj.create_lists_from_default_template package: <<~PACK
CPMAddPackage(
NAME testpack-adder
Expand Down
21 changes: 21 additions & 0 deletions test/integration/test_parallelism.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
require_relative './lib'

class Parallelism < IntegrationTest
def setup
@cache_dir = File.join(cur_test_dir, 'cpmcache')
ENV['CPM_SOURCE_CACHE'] = @cache_dir
end

def test_populate_cache_in_parallel
4.times.map { |i|
prj = make_project name: i.to_s, from_template: 'using-fibadder'
prj.create_lists_from_default_template package: 'CPMAddPackage("gh:cpm-cmake/testpack-fibadder@1.0.0")'
prj
}.map { |prj|
Thread.new do
assert_success prj.configure
assert_success prj.build
end
}.map(&:join)
end
end
4 changes: 2 additions & 2 deletions test/integration/test_remove_source_dir.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

class RemoveSourceDir < IntegrationTest
def test_remove_source_dir
prj = make_project 'using-adder'
prj = make_project from_template: 'using-adder'

prj.create_lists_from_default_template package: <<~PACK
CPMAddPackage(
NAME testpack-adder
Expand Down
2 changes: 1 addition & 1 deletion test/integration/test_simple.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ class Simple < IntegrationTest
ADDER_PACKAGE_NAME = 'testpack-adder'

def test_update_single_package
prj = make_project 'using-adder'
prj = make_project from_template: 'using-adder'
adder_cache0 = nil
adder_ver_file = nil

Expand Down
4 changes: 2 additions & 2 deletions test/integration/test_source_cache.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ def setup
end

def test_add_remove_dependency
prj = make_project 'using-fibadder'
prj = make_project from_template: 'using-fibadder'

###################################
# create
Expand Down Expand Up @@ -45,7 +45,7 @@ def test_add_remove_dependency
end

def test_second_project
prj = make_project 'using-fibadder'
prj = make_project from_template: 'using-fibadder'
prj.create_lists_from_default_template package: 'CPMAddPackage("gh:cpm-cmake/testpack-fibadder@1.1.0")'
assert_success prj.configure

Expand Down
24 changes: 13 additions & 11 deletions test/unit/cache.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,17 @@ function(reset_test)
update_cmake_lists()
endfunction()

function(assert_cache_directory_count directory count)
set(version_count 0)
file(GLOB potential_versions ${directory})
foreach(entry ${potential_versions})
if(IS_DIRECTORY ${entry})
math(EXPR version_count "${version_count} + 1")
endif()
endforeach()
assert_equal("${version_count}" "${count}")
endfunction()

set(FIBONACCI_VERSION 1.0)

# Read CPM_SOURCE_CACHE from arguments
Expand All @@ -40,26 +51,17 @@ execute_process(
assert_equal(${ret} "0")
assert_exists("${CPM_SOURCE_CACHE_DIR}/fibonacci")

file(GLOB FIBONACCI_VERSIONs "${CPM_SOURCE_CACHE_DIR}/fibonacci/*")
list(LENGTH FIBONACCI_VERSIONs FIBONACCI_VERSION_count)
assert_equal(${FIBONACCI_VERSION_count} "1")

file(GLOB fibonacci_versions "${CPM_SOURCE_CACHE_DIR}/fibonacci/*")
list(LENGTH fibonacci_versions fibonacci_version_count)
assert_equal(${fibonacci_version_count} "1")
assert_cache_directory_count("${CPM_SOURCE_CACHE_DIR}/fibonacci/*" 1)

# Update dependency and keep CPM_SOURCE_CACHE

set(FIBONACCI_VERSION 2.0)
update_cmake_lists()

execute_process(COMMAND ${CMAKE_COMMAND} ${TEST_BUILD_DIR} RESULT_VARIABLE ret)

assert_equal(${ret} "0")

file(GLOB FIBONACCI_VERSIONs "${CPM_SOURCE_CACHE_DIR}/fibonacci/*")
list(LENGTH FIBONACCI_VERSIONs FIBONACCI_VERSION_count)
assert_equal(${FIBONACCI_VERSION_count} "2")
assert_cache_directory_count("${CPM_SOURCE_CACHE_DIR}/fibonacci/*" 2)

# Clear cache and update

Expand Down