Compiling: find bin -type l ! -name ".*" -exec rm {} \; ; /home/benchmark/prefix/hojo-3/linux/python-3.9.gcc/99a1a29fd465f86265a7c5b92d75dbcd/bin/python3.9 ./scons.py bin mode=debug cxx=clang extras=tensorflow -j24
Running: unset __PYVENV_LAUNCHER__ && . /home/benchmark/prefix/hojo-3/linux/python_virtual_environments/python-3.9/49faf5ecda3fe7c8dc1e921952eef3eb/bin/activate && cd /home/benchmark/rosetta/tests/integration && python ./integration.py --mode=debug --compiler=clang --extras=tensorflow --timeout=3600 -j24 --skip-comparison --suffix tensorflow --additional_flags "-in:path:database_cache_dir /home/benchmark/rosetta/.database-binaries/tensorflow.linuxclangdebug"
Running integration script...
Command line: unset __PYVENV_LAUNCHER__ && . /home/benchmark/prefix/hojo-3/linux/python_virtual_environments/python-3.9/49faf5ecda3fe7c8dc1e921952eef3eb/bin/activate && cd /home/benchmark/rosetta/tests/integration && python ./integration.py --mode=debug --compiler=clang --extras=tensorflow --timeout=3600 -j24 --skip-comparison --suffix tensorflow --additional_flags "-in:path:database_cache_dir /home/benchmark/rosetta/.database-binaries/tensorflow.linuxclangdebug"
Using Rosetta source dir at: /home/benchmark/rosetta/source
Using Rosetta database dir at:/home/benchmark/rosetta/database
Current Versions Tested:
MAIN: 1f5080a079a5261122c0e532c46f61a4f7e20df8
TOOLS: b0af2521cbed1f2bde597ef1c5406efa376a6f85
DEMOS: 48c9239db73b9a8c828954d9215c983b9f144c92
Python: `/home/benchmark/prefix/hojo-3/linux/python_virtual_environments/python-3.9/49faf5ecda3fe7c8dc1e921952eef3eb/bin/python`
Outdir: new
Running Test trRosetta_test_predict_ubiquitin_init_by_bins
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosetta_test_predict_ubiquitin_init_by_bins/command.tensorflow.sh
Running Test trRosetta_test_predict_ubiquitin_cst_file_write_only
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosetta_test_predict_ubiquitin_cst_file_write_only/command.tensorflow.sh
Running Test trRosetta_test_predict_ubiquitin_cst_file_write
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosetta_test_predict_ubiquitin_cst_file_write/command.tensorflow.sh
Running Test trRosetta_test_predict_ubiquitin
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosetta_test_predict_ubiquitin/command.tensorflow.sh
Running Test trRosetta_test_predict
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosetta_test_predict/command.tensorflow.sh
Running Test trRosettaProtocolMover_rosettascripts_diskwrite_only
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosettaProtocolMover_rosettascripts_diskwrite_only/command.tensorflow.sh
Running Test trRosettaProtocolMover_rosettascripts_diskwrite
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosettaProtocolMover_rosettascripts_diskwrite/command.tensorflow.sh
Running Test trRosettaProtocolMover_rosettascripts
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosettaProtocolMover_rosettascripts/command.tensorflow.sh
Running Test trRosettaProtocolMover
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosettaProtocolMover/command.tensorflow.sh
Running Test trRosettaConstraintGenerator_rosettascripts
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosettaConstraintGenerator_rosettascripts/command.tensorflow.sh
Running Test trRosettaConstraintGenerator
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosettaConstraintGenerator/command.tensorflow.sh
Running Test trRosetta
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/trRosetta/command.tensorflow.sh
Running Test tensorflow_simple_model_load_and_evaluate
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/tensorflow_simple_model_load_and_evaluate/command.tensorflow.sh
Running Test tensorflow_manager
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/tensorflow_manager/command.tensorflow.sh
Running Test tensorflow_connection_test
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/tensorflow_connection_test/command.tensorflow.sh
Running Test smart_annealer
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/smart_annealer/command.tensorflow.sh
Running Test esm_model_perplexity
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/esm_model_perplexity/command.tensorflow.sh
Running Test database_md5
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/database_md5/command.tensorflow.sh
Running Test basic_gcn_tensorflow_test
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/basic_gcn_tensorflow_test/command.tensorflow.sh
Running Test abinitio_with_trRosetta
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/abinitio_with_trRosetta/command.tensorflow.sh
Running Test PTMPrediction
ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/PTMPrediction/command.tensorflow.sh
Finished basic_gcn_tensorflow_test in 2 seconds [~ 803 test (100.0%) started, 0 in queue, 20 running]
grep: mpi_log*: No such file or directory
grep: mpi_log*: No such file or directory
Finished tensorflow_simple_model_load_and_evaluate in 11 seconds [~ 803 test (100.0%) started, 0 in queue, 19 running]
Finished tensorflow_connection_test in 11 seconds [~ 803 test (100.0%) started, 0 in queue, 18 running]
Encounter error while executing: ulimit -t 3600 && bash /home/benchmark/rosetta/tests/integration/new/tensorflow_manager/command.tensorflow.sh
*** Test tensorflow_manager did not run! Check your --mode flag and paths. [2025-01-13 18:27:51.414321]
Finished tensorflow_manager in 12 seconds [~ 803 test (100.0%) started, 0 in queue, 17 running]
Finished trRosettaConstraintGenerator in 35 seconds [~ 803 test (100.0%) started, 0 in queue, 16 running]
Finished trRosetta in 46 seconds [~ 803 test (100.0%) started, 0 in queue, 15 running]
Finished trRosetta_test_predict_ubiquitin_cst_file_write_only in 93 seconds [~ 803 test (100.0%) started, 0 in queue, 14 running]
Finished esm_model_perplexity in 120 seconds [~ 803 test (100.0%) started, 0 in queue, 13 running]
Finished database_md5 in 152 seconds [~ 803 test (100.0%) started, 0 in queue, 12 running]
Finished trRosettaProtocolMover_rosettascripts_diskwrite_only in 173 seconds [~ 803 test (100.0%) started, 0 in queue, 11 running]
Finished PTMPrediction in 604 seconds [~ 803 test (100.0%) started, 0 in queue, 10 running]
Finished smart_annealer in 690 seconds [~ 803 test (100.0%) started, 0 in queue, 9 running]
Finished trRosetta_test_predict in 1499 seconds [~ 803 test (100.0%) started, 0 in queue, 8 running]
Finished trRosettaProtocolMover in 1544 seconds [~ 803 test (100.0%) started, 0 in queue, 7 running]
Finished trRosetta_test_predict_ubiquitin in 1588 seconds [~ 803 test (100.0%) started, 0 in queue, 6 running]
Finished trRosetta_test_predict_ubiquitin_cst_file_write in 1629 seconds [~ 803 test (100.0%) started, 0 in queue, 5 running]
Finished trRosettaConstraintGenerator_rosettascripts in 1646 seconds [~ 803 test (100.0%) started, 0 in queue, 4 running]
Finished trRosetta_test_predict_ubiquitin_init_by_bins in 1803 seconds [~ 803 test (100.0%) started, 0 in queue, 3 running]
Finished trRosettaProtocolMover_rosettascripts in 2248 seconds [~ 803 test (100.0%) started, 0 in queue, 2 running]
Finished trRosettaProtocolMover_rosettascripts_diskwrite in 2679 seconds [~ 803 test (100.0%) started, 0 in queue, 1 running]
Finished abinitio_with_trRosetta in 2862 seconds [~ 803 test (100.0%) started, 0 in queue, 0 running]
Skipping comparison/analysis phase because command line option "--skip-comparison" was specified...
Missing new/runtimes.yaml
──────────────── 'hojo-3' comparing main:62141 linux.clang.tensorflow.integration.tensorflow test_id=847381 vs. main:62140 previous_test_id=846657 ────────────────
{
"compared_with_test": {
"full_name": "linux.clang.tensorflow.integration.tensorflow",
"name": "integration.tensorflow",
"platform": {
"compiler": "clang",
"extras": [
"tensorflow"
],
"os": "linux"
},
"platform_as_string": "linux.clang.tensorflow",
"revision": {
"branch": "main",
"revision_id": 62140
},
"state": "passed",
"test_id": 846657
},
"summary": {
"failed": 1,
"failed_tests": [
"tensorflow_manager"
],
"total": 21
},
"tests": {
"PTMPrediction": {
"log": "",
"state": "passed"
},
"abinitio_with_trRosetta": {
"log": "",
"state": "passed"
},
"basic_gcn_tensorflow_test": {
"log": "",
"state": "passed"
},
"database_md5": {
"log": "",
"state": "passed"
},
"esm_model_perplexity": {
"log": "",
"state": "passed"
},
"smart_annealer": {
"log": "",
"state": "passed"
},
"tensorflow_connection_test": {
"log": "",
"state": "passed"
},
"tensorflow_manager": {
"log": "Brief Diff:\nFiles /home/benchmark/working_dir/main:62140/tensorflow_manager/log and /home/benchmark/working_dir/main:62141/tensorflow_manager/log differ\r\nOnly in /home/benchmark/working_dir/main:62141/tensorflow_manager: ROSETTA_CRASH.log\r\nOnly in /home/benchmark/working_dir/main:62141/tensorflow_manager: .test_did_not_run.log\r\n\n\nFull Diff:\ndiff -r '--exclude=command.sh' '--exclude=command.mpi.sh' '--exclude=observers' '--exclude=*.ignore' /home/benchmark/working_dir/main:62140/tensorflow_manager/log /home/benchmark/working_dir/main:62141/tensorflow_manager/log\r\n74,86c74,110\r\n< apps.pilot.jackmaguire.tensorflow_manager_multi_input_test: Output from model: 0.5004\tExpected: 0.5004\tPASSED\r\n< apps.pilot.jackmaguire.tensorflow_manager_multi_input_test: Output from model: 0.40971\tExpected: 0.40971\tPASSED\r\n< apps.pilot.jackmaguire.tensorflow_manager_multi_input_test: Output from model: 0.4043\tExpected: 0.4043\tPASSED\r\n< apps.pilot.jackmaguire.tensorflow_manager_multi_input_test: Output from model: 0.33565\tExpected: 0.33565\tPASSED\r\n< apps.pilot.jackmaguire.tensorflow_manager_multi_input_test: Output from model: 0.71121\tExpected: 0.71121\tPASSED\r\n< apps.pilot.jackmaguire.tensorflow_manager_multi_input_test: Success!\r\n< basic.citation_manager.CitationManager: \r\n< The following UNPUBLISHED Rosetta modules were used during this run of Rosetta. Their authors should be included in the author list when this work is published:\r\n< \r\n< RosettaTensorflowManager Singleton's author(s):\r\n< Vikram K. Mulligan, Systems Biology, Center for Computational Biology, Flatiron Institute [vmulligan@flatironinstitute.org] (Created the RosettaTensorflowManager.)\r\n< Jack Magure, Menten AI [jack.maguire@menten.ai] (Expanded RosettaTensorflowManager capabilities for multi-head jobs and wrote tests.)\r\n< Sergey Lyskov, Gray Lab, Department of Chemical & Biomolecular Engineering, Johns Hopkins University [Sergey.Lyskov@jhu.edu] (Added testing infrastructure and helped to create the Rosetta-Tensorflow linked build.)\r\n---\r\n> \r\n> ERROR: MISMATCH!!!\tOriginal[4](1):\tnan\tCombined(3+1):\tnan\r\n> MISMATCH!!!\tOriginal[5](1):\tnan\tCombined(4+1):\tnan\r\n> \r\n> \r\n> Error in RosettaTensorflowTensorContainer< T >::combine_tensors(): Failed to copy tensor properly! See error messages above.\r\n> ERROR:: Exit from: src/basic/tensorflow_manager/RosettaTensorflowTensorContainer.tmpl.hh line: 313\r\n> \r\n> [ ERROR ]: Caught exception:\r\n> \r\n> \r\n> File: src/basic/tensorflow_manager/RosettaTensorflowTensorContainer.tmpl.hh:313\r\n> [ ERROR ] UtilityExitException\r\n> ERROR: MISMATCH!!!\tOriginal[4](1):\tnan\tCombined(3+1):\tnan\r\n> MISMATCH!!!\tOriginal[5](1):\tnan\tCombined(4+1):\tnan\r\n> \r\n> \r\n> Error in RosettaTensorflowTensorContainer< T >::combine_tensors(): Failed to copy tensor properly! See error messages above.\r\n> \r\n> \r\n> ------------------------ Begin developer's backtrace ------------------------- \r\n> BACKTRACE:\r\n> ROSETTA/source/build/src/debug/linux/5.4/64/x86/clang/3.4/tensorflow/libutility.so(backtrace_string(int)+0x41) [0x7f108ceacb01]\r\n> ROSETTA/source/build/src/debug/linux/5.4/64/x86/clang/3.4/tensorflow/libutility.so(utility::excn::Exception::Exception(char const*, int, std::string const&)+0xb1) [0x7f108ceed301]\r\n> ROSETTA/source/build/src/debug/linux/5.4/64/x86/clang/3.4/tensorflow/libutility.so(utility::UtilityExitException::UtilityExitException(char const*, int, std::string const&)+0x7e) [0x7f108ceb044e]\r\n> ROSETTA/source/build/src/debug/linux/5.4/64/x86/clang/3.4/tensorflow/libutility.so(utility::exit(char const*, int, std::string const&, int)+0x56) [0x7f108ceb0046]\r\n> ROSETTA/source/bin/tensorflow_manager_multi_input_test.tensorflow.linuxclangdebug(basic::tensorflow_manager::RosettaTensorflowTensorContainer<float>::combine_tensors(utility::vector1<basic::tensorflow_manager::RosettaTensorflowTensorContainer<float>, std::allocator<basic::tensorflow_manager::RosettaTensorflowTensorContainer<float> > > const&, basic::tensorflow_manager::RosettaTensorflowTensorContainer<float>&)+0xd58) [0x426258]\r\n> ROSETTA/source/bin/tensorflow_manager_multi_input_test.tensorflow.linuxclangdebug() [0x4250ad]\r\n> ROSETTA/source/bin/tensorflow_manager_multi_input_test.tensorflow.linuxclangdebug() [0x4240fa]\r\n> ROSETTA/source/bin/tensorflow_manager_multi_input_test.tensorflow.linuxclangdebug() [0x421451]\r\n> ROSETTA/source/bin/tensorflow_manager_multi_input_test.tensorflow.linuxclangdebug() [0x422310]\r\n> /lib64/libc.so.6(__libc_start_main+0xf5) [0x7f107f744555]\r\n> ROSETTA/source/bin/tensorflow_manager_multi_input_test.tensorflow.linuxclangdebug() [0x41dfb3]\r\n> ------------------------- End developer's backtrace -------------------------- \r\n> \r\n> \r\n> AN INTERNAL ERROR HAS OCCURED. PLEASE SEE THE CONTENTS OF ROSETTA_CRASH.log FOR DETAILS.\r\nOnly in /home/benchmark/working_dir/main:62141/tensorflow_manager: ROSETTA_CRASH.log\r\nOnly in /home/benchmark/working_dir/main:62141/tensorflow_manager: .test_did_not_run.log\r\n\nCompare(...): Marking as \"Script failed\" due to presense of .test_did_not_run.log or .test_got_timeout_kill.log file!\n",
"state": "script failed"
},
"tensorflow_simple_model_load_and_evaluate": {
"log": "",
"state": "passed"
},
"trRosetta": {
"log": "",
"state": "passed"
},
"trRosettaConstraintGenerator": {
"log": "",
"state": "passed"
},
"trRosettaConstraintGenerator_rosettascripts": {
"log": "",
"state": "passed"
},
"trRosettaProtocolMover": {
"log": "",
"state": "passed"
},
"trRosettaProtocolMover_rosettascripts": {
"log": "",
"state": "passed"
},
"trRosettaProtocolMover_rosettascripts_diskwrite": {
"log": "",
"state": "passed"
},
"trRosettaProtocolMover_rosettascripts_diskwrite_only": {
"log": "",
"state": "passed"
},
"trRosetta_test_predict": {
"log": "",
"state": "passed"
},
"trRosetta_test_predict_ubiquitin": {
"log": "",
"state": "passed"
},
"trRosetta_test_predict_ubiquitin_cst_file_write": {
"log": "",
"state": "passed"
},
"trRosetta_test_predict_ubiquitin_cst_file_write_only": {
"log": "",
"state": "passed"
},
"trRosetta_test_predict_ubiquitin_init_by_bins": {
"log": "",
"state": "passed"
}
}
}