|
| 1 | +// Copyright (C) 2024-2025 Intel Corporation |
| 2 | +// SPDX-License-Identifier: Apache-2.0 |
| 3 | +// |
| 4 | + |
| 5 | +#include "test_engine/simple_llm_pipeline.hpp" |
| 6 | +#include "test_engine/mocks/mock_plugins.hpp" |
| 7 | +#include "test_engine/mocks/register_in_ov.hpp" |
| 8 | +#include "intel_npu/npuw_private_properties.hpp" |
| 9 | +#include "openvino/util/shared_object.hpp" |
| 10 | +#include "openvino/util/file_util.hpp" |
| 11 | + |
| 12 | +#include <filesystem> |
| 13 | + |
| 14 | +using namespace testing; |
| 15 | +using namespace ov::npuw::tests; |
| 16 | +using namespace ov::intel_npu::npuw; |
| 17 | + |
| 18 | +namespace { |
| 19 | + const std::vector<int64_t> What_is_OpenVINO = |
| 20 | + {529, 29989, 1792, 29989, 29958, 13, 5618, 338, 4673, 29963, 1177, |
| 21 | + 29949, 29973, 2, 29871, 13, 29966, 29989, 465, 22137, 29989, 29958, 13}; |
| 22 | +} // anonymous namespace |
| 23 | + |
| 24 | +#define TIMES(times) times |
| 25 | + |
| 26 | +#define THROW(...) \ |
| 27 | + .WillOnce(Throw(std::runtime_error(__VA_ARGS__))) |
| 28 | + |
| 29 | +#define EXPECT_COMPILE_MODEL(device, times, ...) \ |
| 30 | + EXPECT_CALL(*device##_plugin, \ |
| 31 | + compile_model(A<const std::shared_ptr<const ov::Model>&>(), _)) \ |
| 32 | + .Times(times) \ |
| 33 | + __VA_ARGS__ \ |
| 34 | + |
| 35 | +#define MODEL(idx) idx |
| 36 | + |
| 37 | +#define INFER_REQ(idx) idx |
| 38 | + |
| 39 | +#define EXPECT_CREATE_SYNC_INFER_REQ(device, model_idx, times, ...) \ |
| 40 | + device##_plugin->set_expectations_to_comp_models(model_idx, [](MockCompiledModel& model) { \ |
| 41 | + EXPECT_CALL(model, create_sync_infer_request()) \ |
| 42 | + .Times(times) \ |
| 43 | + __VA_ARGS__; \ |
| 44 | + }) |
| 45 | + |
| 46 | +#define EXPECT_INFER(device, model_idx, times, ...) \ |
| 47 | + device##_plugin->set_expectations_to_infer_reqs(model_idx, 0, [](MockInferRequest& request) { \ |
| 48 | + EXPECT_CALL(request, infer()) \ |
| 49 | + .Times(times) \ |
| 50 | + __VA_ARGS__; \ |
| 51 | + }); |
| 52 | + |
| 53 | +#define EXPECT_INFER_FOR(device, model_idx, req_idx, times, ...) \ |
| 54 | + device##_plugin->set_expectations_to_infer_reqs(model_idx, req_idx, [](MockInferRequest& request) { \ |
| 55 | + EXPECT_CALL(request, infer()) \ |
| 56 | + .Times(times) \ |
| 57 | + __VA_ARGS__; \ |
| 58 | + }); |
| 59 | + |
| 60 | +namespace ov { |
| 61 | +namespace npuw { |
| 62 | +namespace tests { |
| 63 | + |
| 64 | +class LLMBehaviorTestsNPUW : public ::testing::Test { |
| 65 | +public: |
| 66 | + void SetUp() override { |
| 67 | + mock_npu_for_prefill_plugin = std::make_shared<MockNpuPluginForPrefill>(); |
| 68 | + mock_npu_for_prefill_plugin->create_implementation(); |
| 69 | + mock_npu_for_generate_plugin = std::make_shared<MockNpuPluginForGenerate>(); |
| 70 | + mock_npu_for_generate_plugin->create_implementation(); |
| 71 | + config["++NPUW_LLM_PREFILL_CONFIG"] = ov::AnyMap{{"NPUW_DEVICES", "MockNPUForPrefill"}}; |
| 72 | + config["++NPUW_LLM_GENERATE_CONFIG"] = ov::AnyMap{{"NPUW_DEVICES", "MockNPUForGenerate"}}; |
| 73 | + config["NPUW_LLM_MIN_RESPONSE_LEN"] = 2; |
| 74 | + } |
| 75 | + |
| 76 | + // Make sure it is called after expectations are set! |
| 77 | + void register_mock_plugins_in_ov() { |
| 78 | + m_shared_objects.push_back(reg_plugin<MockNpuPluginForPrefill>(core, mock_npu_for_prefill_plugin)); |
| 79 | + m_shared_objects.push_back(reg_plugin<MockNpuPluginForGenerate>(core, mock_npu_for_generate_plugin)); |
| 80 | + } |
| 81 | + |
| 82 | +protected: |
| 83 | + ov::Core core; |
| 84 | + SimpleLLMPipeline simple_llm; |
| 85 | + std::shared_ptr<MockNpuPluginForPrefill> mock_npu_for_prefill_plugin; |
| 86 | + std::shared_ptr<MockNpuPluginForGenerate> mock_npu_for_generate_plugin; |
| 87 | + ov::AnyMap config; |
| 88 | + |
| 89 | +private: |
| 90 | + std::vector<std::shared_ptr<void>> m_shared_objects; |
| 91 | +}; |
| 92 | +} // namespace tests |
| 93 | +} // namespace npuw |
| 94 | +} // namespace ov |
| 95 | + |
| 96 | +TEST_F(LLMBehaviorTestsNPUW, LLMBehaviorNPUW_FAST_COMPILE) { |
| 97 | + // Set expectations first: |
| 98 | + { |
| 99 | + InSequence seq; |
| 100 | + // Generate model: |
| 101 | + EXPECT_COMPILE_MODEL(mock_npu_for_generate, TIMES(3)); |
| 102 | + // Prefill model: |
| 103 | + EXPECT_COMPILE_MODEL(mock_npu_for_prefill, TIMES(3)); |
| 104 | + } |
| 105 | + |
| 106 | + // ------------------------ Prefill model --------------------------- |
| 107 | + // 1 infer request for head: |
| 108 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_prefill, MODEL(0), TIMES(1)); |
| 109 | + // 2 infer requests for function, `create_sync_infer_request()` |
| 110 | + // should be called twice here (2 requests to form pipelining): |
| 111 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_prefill, MODEL(1), TIMES(2)); |
| 112 | + // 1 infer request for tail: |
| 113 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_prefill, MODEL(2), TIMES(1)); |
| 114 | + |
| 115 | + // Head's infer request is called once: |
| 116 | + EXPECT_INFER(mock_npu_for_prefill, MODEL(0), TIMES(1)); |
| 117 | + // There are 20 repeated functions, so: |
| 118 | + // Repeated block's 1st infer request is called 10 times: |
| 119 | + EXPECT_INFER_FOR(mock_npu_for_prefill, MODEL(1), INFER_REQ(0), TIMES(10)); |
| 120 | + // Repeated block's 2nd infer request (brother of 1st one) is called 10 times: |
| 121 | + EXPECT_INFER_FOR(mock_npu_for_prefill, MODEL(1), INFER_REQ(1), TIMES(10)); |
| 122 | + // Tail's infer request is called once: |
| 123 | + EXPECT_INFER(mock_npu_for_prefill, MODEL(2), TIMES(1)); |
| 124 | + |
| 125 | + // ------------------------ Generate model --------------------------- |
| 126 | + // 1 infer request for head: |
| 127 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_generate, MODEL(0), TIMES(1)); |
| 128 | + // 2 infer requests for function, `create_sync_infer_request()` |
| 129 | + // should be called 20 times here (due to UNFOLD_IREQS): |
| 130 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_generate, MODEL(1), TIMES(20)); |
| 131 | + // 1 infer request for tail: |
| 132 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_generate, MODEL(2), TIMES(1)); |
| 133 | + |
| 134 | + // Head's infer request is called once: |
| 135 | + EXPECT_INFER(mock_npu_for_generate, MODEL(0), TIMES(1)); |
| 136 | + // Different 20 infer requests of 2nd submodel should be called: |
| 137 | + for (int i = 0; i < 20; ++i) { |
| 138 | + EXPECT_INFER_FOR(mock_npu_for_generate, MODEL(1), INFER_REQ(i), TIMES(1)); |
| 139 | + } |
| 140 | + // Tail's infer request is called once: |
| 141 | + EXPECT_INFER(mock_npu_for_generate, MODEL(2), TIMES(1)); |
| 142 | + |
| 143 | + // Register mock objects as plugins in OpenVINO: |
| 144 | + register_mock_plugins_in_ov(); |
| 145 | + |
| 146 | + // Do the actual test: |
| 147 | + simple_llm.initialize("C:\\apronina\\models\\TinyLlama-1.1B-Chat-v1.0_int4_sym_group128_dyn_stateful\\TinyLlama-1.1B-Chat-v1.0_int4_sym_group128_dyn_stateful\\openvino_model.xml", |
| 148 | + core, config); |
| 149 | + EXPECT_NO_THROW(simple_llm.generate(What_is_OpenVINO)); |
| 150 | +} |
| 151 | + |
| 152 | +TEST_F(LLMBehaviorTestsNPUW, LLMBehaviorNPUW_BEST_PERF) { |
| 153 | + // Set expectations first: |
| 154 | + { |
| 155 | + InSequence seq; |
| 156 | + // Generate model: |
| 157 | + EXPECT_COMPILE_MODEL(mock_npu_for_generate, TIMES(1)); |
| 158 | + // Prefill model: |
| 159 | + EXPECT_COMPILE_MODEL(mock_npu_for_prefill, TIMES(3)); |
| 160 | + } |
| 161 | + |
| 162 | + // ------------------------ Prefill model --------------------------- |
| 163 | + // 1 infer request for head: |
| 164 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_prefill, MODEL(0), TIMES(1)); |
| 165 | + // 2 infer requests for function, `create_sync_infer_request()` |
| 166 | + // should be called twice here (2 requests to form pipelining): |
| 167 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_prefill, MODEL(1), TIMES(2)); |
| 168 | + // 1 infer request for tail: |
| 169 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_prefill, MODEL(2), TIMES(1)); |
| 170 | + |
| 171 | + // Head's infer request is called once: |
| 172 | + EXPECT_INFER(mock_npu_for_prefill, MODEL(0), TIMES(1)); |
| 173 | + // There are 20 repeated functions, so: |
| 174 | + // Repeated block's 1st infer request is called 10 times: |
| 175 | + EXPECT_INFER_FOR(mock_npu_for_prefill, MODEL(1), INFER_REQ(0), TIMES(10)); |
| 176 | + // Repeated block's 2nd infer request (brother of 1st one) is called 10 times: |
| 177 | + EXPECT_INFER_FOR(mock_npu_for_prefill, MODEL(1), INFER_REQ(1), TIMES(10)); |
| 178 | + // Tail's infer request is called once: |
| 179 | + EXPECT_INFER(mock_npu_for_prefill, MODEL(2), TIMES(1)); |
| 180 | + |
| 181 | + // ------------------------ Generate model --------------------------- |
| 182 | + // 1 infer request for the whole generate model: |
| 183 | + EXPECT_CREATE_SYNC_INFER_REQ(mock_npu_for_generate, MODEL(0), TIMES(1)); |
| 184 | + // Infer request is called only once: |
| 185 | + EXPECT_INFER(mock_npu_for_generate, MODEL(0), TIMES(1)); |
| 186 | + |
| 187 | + // Register mock objects as plugins in OpenVINO: |
| 188 | + register_mock_plugins_in_ov(); |
| 189 | + |
| 190 | + // Do the actual test: |
| 191 | + |
| 192 | + config["NPUW_LLM_GENERATE_HINT"] = "BEST_PERF"; |
| 193 | + simple_llm.initialize("C:\\apronina\\models\\TinyLlama-1.1B-Chat-v1.0_int4_sym_group128_dyn_stateful\\TinyLlama-1.1B-Chat-v1.0_int4_sym_group128_dyn_stateful\\openvino_model.xml", |
| 194 | + core, config); |
| 195 | + EXPECT_NO_THROW(simple_llm.generate(What_is_OpenVINO)); |
| 196 | +} |
| 197 | + |
0 commit comments